都开了一个头了，这个功能不实现我心有不甘。Python不行，那么只好转js继续了。
简单分析
现在worker支持socket，可以用于建立TCP或者TLS连接，成功后仅返回相关reader/writer。
显然，我们只能选择建立TCP连接然后自己实现了。
目前想法是这样，如果我需要检测域名example.com的证书，那么我先本地抓取请求的Client Hello消息做测试，后期再考虑根据协议自己生成。
发送请求后，我们从返回的数据中解析出证书。
TLS握手原理
-
一般发送的消息我们称之为
TLSPlaintext，它的里面会包含有子消息，比如handshake、change_cipher_spec、application_data -
握手消息
handshake会包含有不同类型的子消息，比如Client Hello、Server Hello、Certificates等 -
预期行为是：
- 客户端向指定站点发送
Client Hello消息(或者称之为TLSPlaintext - handshake - ClientHello,以下同类型均省略) - 服务器向客户端发送
Server Hello消息 - 服务器向客户端发送
Certificates消息，客户端解析证书
- 客户端向指定站点发送
抓包Client Hello消息
尝试抓了几次浏览器的包，发觉挺麻烦的，遂写了个nodejs版本。
node req.js www.qq.com 443
运行的结果会打印一个Uint8Array出来。
运行依赖的req.js内容如下：
const https = require('https');
const net = require('net');
const args = process.argv.slice(2);
console.log('args', args);
const PORT = args[1] || 443
const LOCAL_ADDR = args[2] || '127.0.0.1'
const DOMAIN = args[0] || 'www.qq.com'
function startCaptrue(){
const server = net.createServer((socket) => {
socket.on('data', (data) => {
console.log('Client hello msg of sni:', DOMAIN);
let buf = `new Uint8Array([`
data.forEach( val => {
buf += (val + ', ')
})
buf += `])`
console.log(buf);
// console.log('Received data from client: ' + data.toString());
});
setTimeout(() => {
console.log('End after 1 second');
socket.end()
server.close()
}, 1000);
socket.on('end', () => {});
});
server.listen(PORT, LOCAL_ADDR, () => {
console.log(`Server is listening on port ${LOCAL_ADDR} ${PORT}`);
sendMsgToServer()
});
}
function sendMsgToServer(){
const options = {
hostname: DOMAIN,
port: PORT,
path: '/path',
method: 'GET',
agent: new https.Agent({
host: LOCAL_ADDR,
servername: DOMAIN
})
};
const req = https.request(options, (res) => {
res.on('data', (d) => {
process.stdout.write(d);
});
});
req.on('error', (e) => {
// console.error(`请求遇到问题: ${e.message}`);
});
req.end();
}
startCaptrue()
解析报文
先解析TLSPlaintext，再handshake，Certificates。 返回的第一个handshake消息是ServerHello，我们不关注，调到第二个即可。
const OFFSET_PLAINMSG = 5
const OFFSET_HANDSHAKE = 4
const OFFSET_CERTS = 3
const ERROR_NOT_HANDSHAKE = new Error(`plainType should be 22 (handshake)`)
const ERROR_NO_CERT_FOUND = new Error(`no cert found in first 3 handshake msg`)
async function parse({ buffer, offset, size }) {
return new Promise(async (resolve, reject) => {
try {
const maxTry = 3
for (let i = 0; i < 3; i++) {
const plainTxt0 = await parseSSLPlainTxt({ buffer, offset, size })
// console.log(`parse plain msg index: ${i}, msg type: ${plainTxt0.type}`)
if (plainTxt0.type != 22) {
ERROR_NOT_HANDSHAKE.message = `plainType should be 22 (handshake), not ${plainTxt0.type}`
reject(ERROR_NOT_HANDSHAKE)
return
// offset += plainTxt0.size
// size -= plainTxt0.size
// continue
}
const handeshake0 = await parseHandshake(plainTxt0)
if (handeshake0.type != 11) {
// console.log(`handeshake0.type: ${handeshake0.type}`);
// reject(new Error(`handeshakeType should be 2 (server hello), not ${handeshake0.type}`))
// reject(new Error(`handeshakeType should be 11 (certtificates), not ${handeshake1.type}`))
// return
offset += plainTxt0.size
size -= plainTxt0.size
continue
}
const certs = await parseCerts(handeshake0)
// console.log("certs:", certs)
const certContents = { ...certs, offset: certs.offset + OFFSET_CERTS, remark: "certContents" }
let certList = [], countSize = OFFSET_CERTS
while (countSize < certs.size) {
const cert0 = await parseCert(certContents)
// console.log(cert0);
certContents.offset += cert0.size + 3
countSize += cert0.size + 3
certList.push(cert0)
}
resolve(certList)
}
reject(new Error(`cannot find certtificates msg in first ${maxTry} msg`))
} catch (err) {
reject(err)
}
})
}
/**
struct {
ContentType type; 1字节
ProtocolVersion version; 2字节
uint16 length;
opaque fragment[SSLPlaintext.length];
} SSLPlaintext;
*/
async function parseSSLPlainTxt({ buffer, offset, size }) {
return new Promise((resolve, reject) => {
try {
// console.log("--------------- SSLPlaintext");
const len = readInt16(buffer, offset + 3);
const plainType = (buffer[offset + 0] & 0xff)
// console.log("---------------plain type: " + (buffer[offset + 0] & 0xff));
// console.log("---------------plain ver: " + (buffer[offset + 1] & 0xff) + " " + (buffer[offset + 2] & 0xff));
// console.log("---------------plain len: " + len);
const newSize = len + 5;
if (size >= newSize)
resolve({ buffer, offset, size: newSize, type: plainType, remark: "plain" })
else
reject(new Error(`plain txt size overflow! buf size: ${size}, plainTxt size: ${newSize}`))
} catch (error) {
reject(error)
}
});
}
/**
struct {
HandshakeType msg_type; 1字节
uint24 length; 3字节
select(HandshakeType) {
...
case client_hello: ClientHello;
case server_hello: ServerHello;
...
}
body;
} Handshake;
*/
async function parseHandshake({ buffer, offset, size }) {
return new Promise((resolve, reject) => {
try {
offset += OFFSET_PLAINMSG;
// console.log("--------------- SSLPlaintext - Handshake");
const handeshakeType = (buffer[offset + 0] & 0xff)
// console.log("--------------- Handshake type: " + handeshakeType);
const len = readInt24(buffer, offset + 1);
// console.log("--------------- Handshake len: " + len);
const newSize = len + OFFSET_HANDSHAKE;
if (size == newSize + OFFSET_PLAINMSG)
resolve({ buffer, offset, size: newSize, type: handeshakeType, remark: "handshake" })
else
reject(new Error(`Handshake size overflow! plainTxt size: ${size}, Handshake size: ${newSize}`))
} catch (error) {
reject(error)
}
});
}
/*
opaque ASN.1Cert<1..2^24-1>;
struct {
ASN.1Cert certificate_list<1..2^24-1>;
} Certificate;
*/
async function parseCerts({ buffer, offset, size }) {
return new Promise((resolve, reject) => {
try {
// console.log("--------------- SSLPlaintext - Handshake - certs");
offset += OFFSET_HANDSHAKE;
const len = readInt24(buffer, offset);
// console.log("--------------- certs len: " + len);
const newSize = len + OFFSET_CERTS;
if (size == newSize + OFFSET_HANDSHAKE)
resolve({ buffer, offset, size: newSize, remark: "certs" })
else
reject(new Error(`Handshake size overflow! plainTxt size: ${size}, Handshake size: ${newSize}`))
} catch (error) {
reject(error)
}
});
}
function parseCert({ buffer, offset, size }) {
return new Promise((resolve, reject) => {
try {
// offset += OFFSET_CERTS;
// console.log(readInt24(buffer, offset)); // 长度
offset += 3
const tag = buffer[offset]
const lenByte = buffer[offset + 1]
// console.log(offset, tag, lenByte);
if (tag != 48) {
reject(new Error(`cryptobyte_asn1.SEQUENCE should be 48, now it's ${tag}`))
return
}
// ITU-T X.690 section 8.1.3
//
// Bit 8 of the first length byte indicates whether the length is short- or
// long-form.
let length, headerLen // length includes headerLen
if ((lenByte & 0x80) == 0) {
// Short-form length (section 8.1.3.4), encoded in bits 1-7.
length = lenByte + 2
headerLen = 2
} else {
// Long-form length (section 8.1.3.5). Bits 1-7 encode the number of octets
// used to encode the length.
let lenLen = lenByte & 0x7f
if (lenLen == 0 || lenLen > 4 || size < (2 + lenLen)) {
reject(new Error("lenLen == 0 || lenLen > 4 || certs.size < (2 + lenLen)"))
return
}
let len32 = readLenByLen(buffer, offset + 2, lenLen)
// ITU-T X.690 section 10.1 (DER length forms) requires encoding the length
// with the minimum number of octets.
if (len32 < 128) {
reject(new Error("Length should have used short-form encoding."))
return
}
// if (len32 >> ((lenLen - 1) * 8) == 0） {
// return new Error("Leading octet is 0. Length should have been at least one byte shorter.")
// }
headerLen = 2 + lenLen
if (headerLen + len32 > size) {
reject(new Error("Overflow...."))
return
}
length = headerLen + len32
}
// console.log("asn1.length: ", length);
// console.log("asn1.offset: ", offset + headerLen);
resolve({ buffer, offset: offset + headerLen, size: length, remark: "single cert" })
// if (length < 0 || !s.ReadBytes((* []byte)(out), int(length))) {
// return false
// }
} catch (error) {
reject(error)
}
});
}
function readInt16(data, offsetBegin) {
return (data[offsetBegin] & 0xff) << 8 | (data[offsetBegin + 1] & 0xff);
}
function readLenByLen(data, offsetBegin, lenOfLen) {
let result = 0;
for (let index = 0; index < lenOfLen; index++) {
result = result << 8;
result |= data[offsetBegin + index] & 0xff;
}
return result;
}
function readInt24(data, offsetBegin) {
return (data[offsetBegin] & 0xff) << 16 | (data[offsetBegin + 1] & 0xff) << 8
| (data[offsetBegin + 2] & 0xff);
}
尝试
import { connect } from 'cloudflare:sockets';
/*
TLS 1.3 0x0304
TLS 1.2 0x0303
TLS 1.1 0x0302
TLS 1.0 0x0301
SSL 3.0 0x0300
*/
const domainClientMap = {
'nicelee.top' : [
('node', new Uint8Array([22, 3, 1, 1, 104, 1, 0, 1, 100, 3, 3, 70, 100, 233, 164, 97, 15, 253, 128, 232, 50, 97, 9, 228, 247, 233, 57, 45, 121, 129, 81, 157, 110, 60, 170, 135, 164, 138, 172, 55, 12, 225, 133, 32, 82, 241, 77, 149, 163, 47, 177, 165, 174, 235, 14, 77, 157, 42, 158, 90, 23, 250, 6, 65, 60, 93, 148, 171, 27, 250, 231, 18, 83, 39, 18, 0, 0, 118, 19, 2, 19, 3, 19, 1, 192, 47, 192, 43, 192, 48, 192, 44, 0, 158, 192, 39, 0, 103, 192, 40, 0, 107, 0, 163, 0, 159, 204, 169, 204, 168, 204, 170, 192, 175, 192, 173, 192, 163, 192, 159, 192, 93, 192, 97, 192, 87, 192, 83, 0, 162, 192, 174, 192, 172, 192, 162, 192, 158, 192, 92, 192, 96, 192, 86, 192, 82, 192, 36, 0, 106, 192, 35, 0, 64, 192, 10, 192, 20, 0, 57, 0, 56, 192, 9, 192, 19, 0, 51, 0, 50, 0, 157, 192, 161, 192, 157, 192, 81, 0, 156, 192, 160, 192, 156, 192, 80, 0, 61, 0, 60, 0, 53, 0, 47, 0, 255, 1, 0, 0, 165, 0, 0, 0, 16, 0, 14, 0, 0, 11, 110, 105, 99, 101, 108, 101, 101, 46, 116, 111, 112, 0, 11, 0, 4, 3, 0, 1, 2, 0, 10, 0, 12, 0, 10, 0, 29, 0, 23, 0, 30, 0, 25, 0, 24, 0, 35, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 13, 0, 48, 0, 46, 4, 3, 5, 3, 6, 3, 8, 7, 8, 8, 8, 9, 8, 10, 8, 11, 8, 4, 8, 5, 8, 6, 4, 1, 5, 1, 6, 1, 3, 3, 2, 3, 3, 1, 2, 1, 3, 2, 2, 2, 4, 2, 5, 2, 6, 2, 0, 43, 0, 5, 4, 3, 4, 3, 3, 0, 45, 0, 2, 1, 1, 0, 51, 0, 38, 0, 36, 0, 29, 0, 32, 54, 44, 66, 7, 221, 170, 245, 64, 153, 234, 114, 153, 252, 86, 98, 237, 167, 101, 235, 182, 229, 4, 27, 245, 30, 229, 167, 65, 170, 106, 99, 97, ])),
],
'www.qq.com' : [
('node', new Uint8Array([22, 3, 1, 1, 103, 1, 0, 1, 99, 3, 3, 235, 207, 4, 116, 226, 217, 161, 199, 52, 124, 65, 36, 66, 145, 116, 119, 49, 125, 178, 59, 131, 144, 8, 82, 92, 26, 184, 158, 115, 144, 191, 7, 32, 204, 6, 99, 107, 6, 124, 34, 194, 3, 74, 11, 235, 176, 35, 17, 64, 244, 52, 44, 83, 158, 97, 251, 205, 61, 59, 60, 10, 169, 131, 2, 221, 0, 118, 19, 2, 19, 3, 19, 1, 192, 47, 192, 43, 192, 48, 192, 44, 0, 158, 192, 39, 0, 103, 192, 40, 0, 107, 0, 163, 0, 159, 204, 169, 204, 168, 204, 170, 192, 175, 192, 173, 192, 163, 192, 159, 192, 93, 192, 97, 192, 87, 192, 83, 0, 162, 192, 174, 192, 172, 192, 162, 192, 158, 192, 92, 192, 96, 192, 86, 192, 82, 192, 36, 0, 106, 192, 35, 0, 64, 192, 10, 192, 20, 0, 57, 0, 56, 192, 9, 192, 19, 0, 51, 0, 50, 0, 157, 192, 161, 192, 157, 192, 81, 0, 156, 192, 160, 192, 156, 192, 80, 0, 61, 0, 60, 0, 53, 0, 47, 0, 255, 1, 0, 0, 164, 0, 0, 0, 15, 0, 13, 0, 0, 10, 119, 119, 119, 46, 113, 113, 46, 99, 111, 109, 0, 11, 0, 4, 3, 0, 1, 2, 0, 10, 0, 12, 0, 10, 0, 29, 0, 23, 0, 30, 0, 25, 0, 24, 0, 35, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 13, 0, 48, 0, 46, 4, 3, 5, 3, 6, 3, 8, 7, 8, 8, 8, 9, 8, 10, 8, 11, 8, 4, 8, 5, 8, 6, 4, 1, 5, 1, 6, 1, 3, 3, 2, 3, 3, 1, 2, 1, 3, 2, 2, 2, 4, 2, 5, 2, 6, 2, 0, 43, 0, 5, 4, 3, 4, 3, 3, 0, 45, 0, 2, 1, 1, 0, 51, 0, 38, 0, 36, 0, 29, 0, 32, 23, 83, 254, 162, 134, 173, 234, 149, 101, 232, 126, 150, 56, 89, 245, 50, 143, 88, 133, 145, 105, 249, 200, 81, 184, 119, 234, 55, 129, 121, 229, 37, ])),
('firefox', new Uint8Array([22, 3, 3, 2, 141, 1, 0, 2, 137, 3, 3, 228, 40, 157, 142, 90, 54, 251, 191, 196, 32, 103, 161, 117, 152, 64, 1, 117, 207, 15, 206, 26, 72, 240, 246, 203, 250, 33, 99, 185, 174, 187, 157, 32, 62, 102, 3, 145, 235, 33, 28, 219, 159, 191, 38, 199, 60, 212, 20, 21, 138, 89, 104, 54, 80, 152, 136, 126, 73, 67, 100, 199, 128, 46, 233, 134, 0, 34, 19, 1, 19, 3, 19, 2, 192, 43, 192, 47, 204, 169, 204, 168, 192, 44, 192, 48, 192, 10, 192, 9, 192, 19, 192, 20, 0, 156, 0, 157, 0, 47, 0, 53, 1, 0, 2, 30, 0, 0, 0, 15, 0, 13, 0, 0, 10, 119, 119, 119, 46, 113, 113, 46, 99, 111, 109, 0, 23, 0, 0, 255, 1, 0, 1, 0, 0, 10, 0, 14, 0, 12, 0, 29, 0, 23, 0, 24, 0, 25, 1, 0, 1, 1, 0, 11, 0, 2, 1, 0, 0, 35, 0, 0, 0, 16, 0, 14, 0, 12, 2, 104, 50, 8, 104, 116, 116, 112, 47, 49, 46, 49, 0, 5, 0, 5, 1, 0, 0, 0, 0, 0, 34, 0, 10, 0, 8, 4, 3, 5, 3, 6, 3, 2, 3, 0, 51, 0, 107, 0, 105, 0, 29, 0, 32, 7, 82, 233, 155, 127, 67, 155, 244, 40, 15, 230, 241, 17, 92, 223, 89, 157, 33, 120, 59, 144, 184, 70, 26, 9, 102, 122, 195, 226, 233, 67, 124, 0, 23, 0, 65, 4, 72, 126, 91, 137, 201, 9, 176, 73, 44, 168, 34, 233, 206, 4, 0, 88, 239, 67, 30, 195, 38, 206, 213, 171, 0, 130, 45, 245, 211, 226, 255, 234, 233, 82, 111, 177, 253, 115, 160, 55, 164, 135, 18, 147, 106, 105, 253, 200, 71, 56, 235, 95, 89, 102, 152, 249, 174, 215, 12, 8, 64, 57, 131, 211, 0, 43, 0, 5, 4, 3, 4, 3, 3, 0, 13, 0, 24, 0, 22, 4, 3, 5, 3, 6, 3, 8, 4, 8, 5, 8, 6, 4, 1, 5, 1, 6, 1, 2, 3, 2, 1, 0, 45, 0, 2, 1, 1, 0, 28, 0, 2, 64, 1, 254, 13, 1, 25, 0, 0, 1, 0, 3, 116, 0, 32, 87, 129, 3, 3, 34, 121, 1, 220, 95, 172, 9, 84, 4, 131, 185, 138, 164, 81, 23, 170, 18, 39, 67, 120, 148, 32, 247, 5, 196, 133, 223, 67, 0, 239, 212, 201, 27, 5, 224, 191, 194, 152, 63, 160, 252, 24, 146, 202, 65, 162, 151, 27, 26, 98, 10, 132, 137, 86, 79, 170, 237, 46, 93, 244, 15, 1, 43, 91, 156, 59, 167, 246, 22, 126, 1, 42, 224, 153, 80, 71, 105, 16, 20, 205, 58, 153, 75, 189, 17, 99, 186, 116, 73, 165, 227, 194, 174, 127, 29, 216, 35, 109, 255, 37, 152, 113, 58, 145, 250, 155, 27, 236, 143, 23, 102, 22, 185, 204, 78, 72, 49, 236, 244, 221, 140, 139, 144, 136, 116, 156, 176, 139, 27, 108, 63, 235, 254, 215, 209, 250, 238, 195, 124, 248, 162, 205, 99, 84, 59, 42, 254, 61, 56, 8, 61, 133, 106, 200, 169, 40, 208, 84, 242, 219, 22, 144, 162, 173, 59, 35, 111, 215, 144, 77, 77, 28, 6, 159, 140, 190, 173, 198, 75, 77, 245, 243, 227, 40, 121, 50, 57, 139, 81, 9, 9, 74, 50, 103, 96, 144, 124, 193, 179, 78, 62, 125, 237, 83, 149, 116, 150, 194, 32, 134, 172, 20, 200, 244, 158, 215, 64, 237, 153, 24, 31, 134, 73, 24, 155, 150, 39, 206, 125, 111, 184, 168, 62, 110, 106, 6, 39, 8, 95, 182, 50, 250, 85, 61, 191, 230, 133, 238, 177, 86, 159, 172, 245, 245, 58, 129, 34, 62, 68, 104, 147, 36, 216, 240, 35, 202, 80, 131, 35])),
]
}
const testIp = null || ''
const testDomain = "www.qq.com"
const testHost = testIp || testDomain
const testPort = 443
const testClientHello = domainClientMap[testDomain]
console.log(`testDomain is ${testDomain}, testHost is ${testHost}, testPort is ${testPort}, `)
export default {
async fetch(req, env, _ctx) {
try {
const address = {
hostname: testHost,
port: testPort
};
const socket = connect(address);
const writer = socket.writable.getWriter()
const reader = socket.readable.getReader()
if(!testClientHello)
throw new Error(`${testDomain} not in domainClientMap`)
await writer.write(testClientHello[0])
const certs = await parseResponse(reader)
// console.log("certs: ", certs)
if(certs === ERROR_NOT_HANDSHAKE || certs === ERROR_NO_CERT_FOUND)
throw certs
const cert = certs[0]
cert.buffer = null
cert.domain = testDomain
//const certInfo = decodeCert(cert.buffer.subarray(cert.offset, cert.offset + cert.size))
socket.close();
return new Response(JSON.stringify(cert), { headers: { "Content-Type": "application/json" } });
} catch (error) {
console.log(error)
return new Response(JSON.stringify({ error: error.message }), { status: 500, headers: { "Content-Type": "application/json" } });
}
}
}
async function parseResponse(reader) {
const buffer = new ArrayBuffer(10240)
const result = new Uint8Array(buffer, 0, buffer.byteLength)
let bytesReceived = 0
while (true) {
let { done, value } = await reader.read()
// console.log("reader.read(): ", done, value);
if (done || bytesReceived > 5120 || !value || value.byteLength == 0 || bytesReceived + value.byteLength >= 10240)
break
result.set(value, bytesReceived)
bytesReceived += value.byteLength
try {
let certs = await parse({
buffer: result,
offset: 0,
size: bytesReceived
})
return certs
} catch (error) {
// console.log(error);
if(error === ERROR_NOT_HANDSHAKE)
return error
continue
}
}
return ERROR_NO_CERT_FOUND
}
... 解析报文的代码同上，省略
结果
本地wrangler测试没有问题，deploy之后发现服务器后续返回不是预期的handshake - certificates，而是change_cipher_spec加上application_data。
阿哲，继续的话还要深入解析TLS协议的不同版本和加密套件。折腾了好久不想搞了。。。
只能说有生之年吧。
后续
突发奇想，能不能在在建立TCP连接后，把socket的readable stream给复制一份，然后再来 starttls套上TLS连接。 可惜，还是报错。
import { connect } from 'cloudflare:sockets';
const testIp = null || ''
const testDomain = "www.qq.com"
const testHost = testIp || testDomain
const testPort = 443
console.log("----- reloading -----");
export default {
async fetch(req, env, _ctx) {
try {
const cert = await testTee()
return new Response(JSON.stringify(cert), { headers: { "Content-Type": "application/json" } });
} catch (error) {
console.log(error)
return new Response(JSON.stringify({ error: error.message }), { status: 500, headers: { "Content-Type": "application/json" } });
}
}
}
async function testTee() {
return new Promise(async (resolve, reject) => {
try {
const address = { hostname: testHost, port: testPort }
let socket = connect(address, { secureTransport: "starttls" })
const [r1, r2] = socket.readable.tee()
let socket2 = { readable: r1, __proto__: socket }
// socket2.startTls = socket.startTls.bind(socket2)
r2.getReader().read().then(e => resolve(e)).catch(e => reject(e))
const secureSocket = socket2.startTls()
setTimeout(()=> reject(Error("timeout")), 2000)
} catch (err) {
reject(err)
}
});
}
报错：
TypeError: Illegal invocation: function called with incorrect `this` reference. See https://developers.cloudflare.com/workers/observability/errors/#illegal-invocation-errors for details
看样子是底层实现的问题，在js层面已经改不动了😳