I was following an article about writing a socket server from scratch, and its mostly working with small frames / packages, but when I try to send about 2kb of data, I get this error:
internal/buffer.js:77
throw new ERR_OUT_OF_RANGE(type || 'offset',
^
RangeError [ERR_OUT_OF_RANGE]: The value of "offset" is out of range. It must be >= 0 and <= 7. Receive
d 8
at boundsError (internal/buffer.js:77:9)
at Buffer.readUInt8 (internal/buffer.js:243:5)
at pm (/home/users/me/main.js:277:24)
at Socket.<anonymous> (/home/users/me/main.js:149:15)
at Socket.emit (events.js:315:20)
at addChunk (_stream_readable.js:297:12)
at readableAddChunk (_stream_readable.js:273:9)
at Socket.Readable.push (_stream_readable.js:214:10)
at TCP.onStreamRead (internal/stream_base_commons.js:186:23) {
code: 'ERR_OUT_OF_RANGE'
}
Here's my server code (some details were changed for security, but here it is in its entirety for the line numbers etc.) but the relevant part here is the function pm [=parseMessage](towards the bottom):
let http = require('http'),
ch = require("child_process"),
crypto = require("crypto"),
fs = require("fs"),
password = fs.readFileSync(“./secretPasswordFile.txt”),
callbacks = {
CHANGEDforSecUrITY(m, cs) {
if(m.password === password) {
if(m.command) {
try {
cs.my = ch.exec(
m.command,
(
err,
stdout,
stderr
) => {
cs.write(ans(s({
err,
stdout,
stderr
})));
}
);
} catch(e) {
cs.write(ans(
s({
error: e.toString()
})
))
}
}
if(m.exit) {
console.log("LOL", cs.my);
if(cs.my && typeof cs.my.kill === "function") {
cs.my.kill();
console.log(cs.my, "DID?");
}
}
cs.write(
ans(
s({
hi: 2,
youSaid:m
}))
)
} else {
cs.write(ans(s({
hey: "wrong password!!"
})))
}
console.log("hi!",m)
}
},
banned = [
"61.19.71.84"
],
server = http.createServer(
(q,r)=> {
if(banned.includes(q.connection.remoteAddress)) {
r.end("Hey man, " + q.connection.remoteAddress,
"I know you're there!!");
} else {
ch.exec(`sudo "$(which node)" -p "console.log(4)"`)
console.log(q.url)
console.log(q.connection.remoteAddress,q.connection.remotePort)
let path = q.url.substring(1)
q.url == "/" &&
(path = "index.html")
q.url == "/secret" &&
(path = "../main.js")
fs.readFile(
"./static/" + path,
(er, f) => {
if(er) {
r.end("<h2>404!!</h2>");
} else {
r.end(f);
}
}
)
}
}
)
server.listen(
process.env.PORT || 80,
c=> {
console.log(c,"helo!!!")
server.on("upgrade", (req, socket) => {
if(req.headers["upgrade"] !== "websocket") {
socket.end("HTTP/1.1 400 Bad Request");
return;
}
let key = req.headers["sec-websocket-key"];
if(key) {
let hash = gav(key)
let headers = [
"HTTP/1.1 101 Web Socket Protocol Handshake",
"Upgrade: WebSocket",
"Connection: Upgrade",
`Sec-WebSocket-Accept: ${hash}`
];
let protocol = req.headers[
"sec-websocket-protocol"
];
let protocols = (
protocol &&
protocol.split(",")
.map(s => s.trim())
|| []
);
protocols.includes("json") &&
headers
.push("Sec-WebSocket-Protocol: json");
let headersStr = (
headers.join("\r\n") +
"\r\n\r\n"
)
console.log(
"Stuff happening",
req.headers,
headersStr
);
fs.writeFileSync("static/logs.txt",headersStr);
socket.write(
headersStr
);
socket.write(ans(JSON.stringify(
{
hello: "world!!!"
}
)))
}
socket.on("data", buf => {
let msg = pm(buf);
console.log("HEY MAN!",msg)
if(msg) {
console.log("GOT!",msg);
for(let k in msg) {
if(callbacks[k]) {
callbacks[k](
msg[k],
socket
)
}
}
} else {
console.log("nope");
}
});
});
}
)
function pm(buf) {
/*
*structure of first byte:
1: if its the last frame in buffer
2 - 4: reserved bits
5 - 8: a number which shows what type of message it is. Chart:
"0": means we continue
"1": means this frame contains text
"2": means this is binary
"0011"(3) - "0111" (11): reserved values
"1000"(8): means connection closed
"1001"(9): ping (checking for response)
"1010"(10): pong (response verified)
"1010"(11) - "1111"(15): reserved for "control" frames
structure of second byte:
1: is it "masked"
2 - 8: length of payload, if less than 126.
if 126, 2 additional bytes are added
if 127 (or more), 6 additional bytes added (total 8)
* */
const myFirstByte = buf.readUInt8(0);
const isThisFinalFrame = isset(myFirstByte,7) //first bit
const [
reserved1,
reserved2,
reserved3
] = [
isset(myFirstByte, 6),
isset(myFirstByte, 5),
isset(myFirstByte, 4) //reserved bits
]
const opcode = myFirstByte & parseInt("1111",2); //checks last 4 bits
//check if closed connection ("1000"(8))
if(opcode == parseInt("1000", 2))
return null; //shows that connection closed
//look for text frame ("0001"(1))
if(opcode == parseInt("0001",2)) {
const theSecondByte = buf.readUInt8(1);
const isMasked = isset(theSecondByte, 7) //1st bit from left side
let currentByteOffset = 2; //we are theSecondByte now, so 2
let payloadLength = theSecondByte & 127; //chcek up to 7 bits
if(payloadLength > 125) {
if(payloadLength === 126) {
payloadLength = buf.readUInt16BE(
currentByteOffset
) //read next two bytes from position
currentByteOffset += 2; //now we left off at
//the fourth byte, so thats where we are
} else {
//if only the second byte is full,
//that shows that there are 6 more
//bytes to hold the length
const right = buf.readUInt32BE(
currentByteOffset
);
const left = buf.readUInt32BE(
currentByteOffset + 4 //the 8th byte ??
);
throw new Error("brutal " + currentByteOffset);
}
}
//if we have masking byte set to 1, get masking key
//
//
//now that we have the lengths
//and possible masks, read the rest
//of the bytes, for actual data
const data = Buffer.alloc(payloadLength);
if(isMasked) {
//can't just copy it,
//have to do some stuff with
//the masking key and this thing called
//"XOR" to the data. Complicated
//formulas, llook into later
//
let maskingBytes = Buffer.allocUnsafe(4);
buf.copy(
maskingBytes,
0,
currentByteOffset,
currentByteOffset + 4
);
currentByteOffset += 4;
for(
let i = 0;
i < payloadLength;
++i
) {
const source = buf.readUInt8(
currentByteOffset++
);
//now mask the source with masking byte
data.writeUInt8(
source ^ maskingBytes[i & 3],
i
);
}
} else {
//just copy bytes directly to our buffer
buf.copy(
data,
0,
currentByteOffset++
);
}
//at this point we have the actual data, so make a json
//
const json = data.toString("utf8");
return p(json);
} else {
return "LOL IDK?!";
}
}
function p(str) {
try {
return JSON.parse(str);
} catch(e){
return str
}
}
function s(ob) {
try {
return JSON.stringify(ob);
} catch(e) {
return e.toString();
}
}
function ans(str) {
const byteLength = Buffer.byteLength(str);
const lengthByteCount = byteLength < 126 ? 0 : 2;
const payloadLength = lengthByteCount === 0 ? byteLength : 126;
const buffer = Buffer.alloc(
2 +
lengthByteCount +
byteLength
);
buffer.writeUInt8(
parseInt("10000001",2), //opcode is "1", at firstbyte
0
);
buffer.writeUInt8(payloadLength, 1); //at second byte
let currentByteOffset = 2; //already wrote second byte by now
if(lengthByteCount > 0) {
buffer.writeUInt16BE(
byteLength,
2 //more length at 3rd byte position
);
currentByteOffset += lengthByteCount; //which is 2 more bytes
//of length, since not supporting more than that
}
buffer.write(str, currentByteOffset); //the rest of the bytes
//are the actual data, see chart in function pm
//
return buffer;
}
function gav(ak) {
return crypto
.createHash("sha1")
.update(ak +'258EAFA5-E914-47DA-95CA-C5AB0DC85B11', "binary")
.digest("base64")
}
function isset(b, k) {
return !!(
b >>> k & 1
)
}
Given that this error does not happen with smaller packets, I'm taking an educated guess that this is due to the limitations of the code here, as mentioned in the offical RFC documentation:
5.4. Fragmentation
The primary purpose of fragmentation is to allow sending a message that is of unknown size when the message is started without having to buffer that message. If messages couldn't be fragmented, then an
endpoint would have to buffer the entire message so its length could
be counted before the first byte is sent. With fragmentation, a
server or intermediary may choose a reasonable size buffer and, when
the buffer is full, write a fragment to the network.A secondary use-case for fragmentation is for multiplexing, where it is not desirable for a large message on one logical channel to
monopolize the output channel, so the multiplexing needs to be free to split the message into smaller fragments to better share the output channel. (Note that the multiplexing extension is not described in this document.)Unless specified otherwise by an extension, frames have no semantic meaning. An intermediary might coalesce and/or split frames, if no
extensions were negotiated by the client and the server or if some
extensions were negotiated, but the intermediary understood all the
extensions negotiated and knows how to coalesce and/or split frames
in the presence of these extensions. One implication of this is that in absence of extensions, senders and receivers must not depend on
the presence of specific frame boundaries.The following rules apply to fragmentation:
o An unfragmented message consists of a single frame with the FIN bit set (Section 5.2) and an opcode other than 0.
o A fragmented message consists of a single frame with the FIN bit clear and an opcode other than 0, followed by zero or more frames with the FIN bit clear and the opcode set to 0, and terminated by a single frame with the FIN bit set and an opcode of 0. A fragmented message is conceptually equivalent to a single larger message whose payload is equal to the concatenation of the payloads of the fragments in order; however, in the presence of extensions, this may not hold true as the extension defines the interpretation of the "Extension data" present. For instance, "Extension data" may only be present at the beginning of the first fragment and apply to subsequent fragments, or there may be "Extension data" present in each of the fragments that applies only to that particular fragment. In the absence of "Extension data", the following example demonstrates how fragmentation works.
EXAMPLE: For a text message sent as three fragments, the first fragment would have an opcode of 0x1 and a FIN bit clear, the second fragment would have an opcode of 0x0 and a FIN bit clear, and the third fragment would have an opcode of 0x0 and a FIN bit that is set.o Control frames (see Section 5.5) MAY be injected in the middle of a fragmented message. Control frames themselves MUST NOT be fragmented.
o Message fragments MUST be delivered to the recipient in the order sent by the sender. o The fragments of one message MUST NOT be interleaved between the fragments of another message unless an extension has been negotiated that can interpret the interleaving.
o An endpoint MUST be capable of handling control frames in the middle of a fragmented message.
o A sender MAY create fragments of any size for non-control messages.
o Clients and servers MUST support receiving both fragmented and unfragmented messages.
o As control frames cannot be fragmented, an intermediary MUST NOT attempt to change the fragmentation of a control frame.
o An intermediary MUST NOT change the fragmentation of a message if any reserved bit values are used and the meaning of these values is not known to the intermediary.
o An intermediary MUST NOT change the fragmentation of any message in the context of a connection where extensions have been negotiated and the intermediary is not aware of the semantics of the negotiated extensions. Similarly, an intermediary that didn't see the WebSocket handshake (and wasn't notified about its content) that resulted in a WebSocket connection MUST NOT change the fragmentation of any message of such connection.
o As a consequence of these rules, all fragments of a message are of the same type, as set by the first fragment's opcode. Since control frames cannot be fragmented, the type for all fragments in a message MUST be either text, binary, or one of the reserved opcodes.
NOTE: If control frames could not be interjected, the latency of a ping, for example, would be very long if behind a large message.
Hence, the requirement of handling control frames in the middle of a
fragmented message.IMPLEMENTATION NOTE: In the absence of any extension, a receiver
doesn't have to buffer the whole frame in order to process it. For
example, if a streaming API is used, a part of a frame can be
delivered to the application. However, note that this assumption
might not hold true for all future WebSocket extensions.
In the words of the article above:
Alignment of Node.js socket buffers with WebSocket message frames
Node.js socket data (I’m talking about net.Socket in this case, not WebSockets) is received in buffered chunks. These are split apart with no regard for where your WebSocket frames begin or end!
What this means is that if your server is receiving large messages fragmented into multiple WebSocket frames, or receiving large numbers of messages in rapid succession, there’s no guarantee that each data buffer received by the Node.js socket will align with the start and end of the byte data that makes up a given frame.
So, as you’re parsing each buffer received by the socket, you’ll need to keep track of where one frame ends and where the next begins. You’ll need to be sure that you’ve received all of the bytes of data for a frame — before you can safely consume that frame’s data.
It may be that one frame ends midway through the same buffer in which the next frame begins. It also may be that a frame is split across several buffers that will be received in succession.
The following diagram is an exaggerated illustration of the issue. In most cases, frames tend to fit inside a buffer. Due to the way the data arrives, you’ll often find that a frame will start and end in line with the start and end of the socket buffer. But this can’t be relied upon in all cases, and must be considered during implementation.
This can take some work to get right.
For the basic implementation that follows below, I have skipped any code for handling large messages or messages split across multiple frames.
So my problem here is that the article skipped the fragmentation code, which is kind of what I need to know... but in that RFC documentation, some examples of fragmentated and unfragmented packets are given:
5.6. Data Frames
Data frames (e.g., non-control frames) are identified by opcodes
where the most significant bit of the opcode is 0. Currently defined opcodes for data frames include 0x1 (Text), 0x2 (Binary). Opcodes
0x3-0x7 are reserved for further non-control frames yet to be
defined.Data frames carry application-layer and/or extension-layer data. The opcode determines the interpretation of the data:
Text
The "Payload data" is text data encoded as UTF-8. Note that a particular text frame might include a partial UTF-8 sequence; however, the whole message MUST contain valid UTF-8. Invalid UTF-8 in reassembled messages is handled as described in Section 8.1.Binary
The "Payload data" is arbitrary binary data whose interpretation is solely up to the application layer.5.7. Examples
o A single-frame unmasked text message
* 0x81 0x05 0x48 0x65 0x6c 0x6c 0x6f (contains "Hello")o A single-frame masked text message
* 0x81 0x85 0x37 0xfa 0x21 0x3d 0x7f 0x9f 0x4d 0x51 0x58 (contains "Hello")o A fragmented unmasked text message
* 0x01 0x03 0x48 0x65 0x6c (contains "Hel") * 0x80 0x02 0x6c 0x6f (contains "lo")o Unmasked Ping request and masked Ping response
* 0x89 0x05 0x48 0x65 0x6c 0x6c 0x6f (contains a body of "Hello", but the contents of the body are arbitrary) * 0x8a 0x85 0x37 0xfa 0x21 0x3d 0x7f 0x9f 0x4d 0x51 0x58 (contains a body of "Hello", matching the body of the ping)o 256 bytes binary message in a single unmasked frame
* 0x82 0x7E 0x0100 [256 bytes of binary data]o 64KiB binary message in a single unmasked frame
* 0x82 0x7F 0x0000000000010000 [65536 bytes of binary data]
So it would appear that is an example of a fragment.
Also this seems relevant:
6.2. Receiving Data
To receive WebSocket data, an endpoint listens on the underlying
network connection. Incoming data MUST be parsed as WebSocket frames as defined in Section 5.2. If a control frame (Section 5.5) is
received, the frame MUST be handled as defined by Section 5.5. Upon
receiving a data frame (Section 5.6), the endpoint MUST note the
/type/ of the data as defined by the opcode (frame-opcode) from
Section 5.2. The "Application data" from this frame is defined as
the /data/ of the message. If the frame comprises an unfragmented
message (Section 5.4), it is said that A WebSocket Message Has Been
Received with type /type/ and data /data/. If the frame is part of
a fragmented message, the "Application data" of the subsequent data
frames is concatenated to form the /data/. When the last fragment is received as indicated by the FIN bit (frame-fin), it is said that A
WebSocket Message Has Been Received with data /data/ (comprised of
the concatenation of the "Application data" of the fragments) and type /type/ (noted from the first frame of the fragmented message).
Subsequent data frames MUST be interpreted as belonging to a new
WebSocket message.Extensions (Section 9) MAY change the semantics of how data is read, specifically including what comprises a message boundary.
Extensions, in addition to adding "Extension data" before the
"Application data" in a payload, MAY also modify the "Application
data" (such as by compressing it).
The problem:
I don't know how to check for fragments and line them up with the node buffers, as mentioned in the article, I'm only able to read very small buffer amounts.
How can I parse larger data chunks using the fragmentation methods mentioned in the RFC documentation and the lining-up of nodeJS buffers alluded to (but not explained) in the article?
from native websocket api NodeJS for larger messages?
No comments:
Post a Comment