4
3
Fork
You've already forked PipePipeExtractor
3

[BilibiliChannelExtractor] random User Agent and refactor #3

Merged
NullPointerException merged 8 commits from chr_56/PipePipeExtractor:random_device into main 2023年12月15日 06:59:14 +01:00
Contributor
Copy link
  • Use random User Agent to reduce risks of "Risk Management" wall
  • Refactor: extract class DeviceForger
  • Refactor: move getUserVideos to BilibiliChannelExtractor
  • Refactor: cleanup BilibiliChannelExtractor
- Use random User Agent to reduce risks of "Risk Management" wall - Refactor: extract class DeviceForger - Refactor: move `getUserVideos` to `BilibiliChannelExtractor` - Refactor: cleanup `BilibiliChannelExtractor`
Author
Contributor
Copy link

Note: there are still few chances to get blocked by "Risk Management" but very rare to happen.

Note: there are still few chances to get blocked by "Risk Management" but very rare to happen.
chr_56 changed title from (削除) [BiliBili] random User Agent and refactor (削除ここまで) to [BilibiliChannelExtractor] random User Agent and refactor 2023年12月07日 04:21:51 +01:00
Author
Contributor
Copy link

However, the dm_img_list (mouse & keyboard actions record) has not been forged.

This may be not working in next days.


Sample of dm_img_list:

%7B%22x%22:302,%22y%22:395,%22z%22:0,%22timestamp%22:73,%22type%22:0%7D,%7B%22x%22:396,%22y%22:393,%22z%22:37,%22timestamp%22:192,%22type%22:0%7D,%7B%22x%22:724,%22y%22:491,%22z%22:158,%22timestamp%22:332,%22type%22:0%7D

URL decoded (Json Array without brackets):

{"x":302,"y":395,"z":0,"timestamp":73,"type":0},{"x":396,"y":393,"z":37,"timestamp":192,"type":0},{"x":724,"y":491,"z":158,"timestamp":332,"type":0}

dm_img_list is generated in https://s1.hdslb.com/bfs/static/jinkela/space/9.space.85c18df037f33e9da25b31e5462b10906c75adfc.js as userLog:

function queryUserLog:

 function(t) {
 var e, n, r = t.preTime, o = t.startTime, i = t.endTime;
 e = r ? this.getPreTimeLog(r) : this.getLog(o, i);
 try {
 n = JSON.stringify(e)
 } catch (t) {
 n = "[]"
 }
 return {
 userLog: e,
 userLogStr: n,
 userAgent: navigator.userAgent,
 webglStr: this.webglStr,
 webglVendorAndRenderer: this.webglVendorAndRenderer
 }

We can see this is accumulated.

function getLog:

 function(t, e) {
 return this.activityDetector.getLog(t, e)
 }

And this may be desugared activityDetector class

[{
 key: "initLogger",
 value: function() {
 var t = this;
 return Object(c.b)(this.samplingTime, (function(e) {
 var n, r, i, c, u, s, l, f;
 if (e instanceof MouseEvent)
 n = {
 type: e.type,
 x: e.clientX,
 y: e.clientY,
 preX: (null === (r = t.preEvent) || void 0 === r ? void 0 : r.x) || 0,
 preY: (null === (i = t.preEvent) || void 0 === i ? void 0 : i.y) || 0,
 changeDistance: t.countDistance && (null === (c = t.preEvent) || void 0 === c ? void 0 : c.x) && (null === (u = t.preEvent) || void 0 === u ? void 0 : u.y) && o(e.clientX, e.clientY, null === (s = t.preEvent) || void 0 === s ? void 0 : s.x, null === (l = t.preEvent) || void 0 === l ? void 0 : l.y) || 0,
 timestamp: t.startTime && Date.now() - t.startTime || 0,
 preType: null === (f = t.preEvent) || void 0 === f ? void 0 : f.type
 },
 e instanceof WheelEvent && (n.deltaX = e.deltaX,
 n.deltaY = e.deltaY);
 else if (e instanceof KeyboardEvent) {
 var p, d;
 n = {
 type: a.Keydown,
 key: e.key,
 preKey: (null === (p = t.preEvent) || void 0 === p ? void 0 : p.key) || "mouse",
 preType: null === (d = t.preEvent) || void 0 === d ? void 0 : d.type,
 timestamp: t.startTime && Date.now() - t.startTime || 0
 }
 } else if (e instanceof FocusEvent) {
 var h;
 n = {
 type: a.Focus,
 preType: null === (h = t.preEvent) || void 0 === h ? void 0 : h.type,
 timestamp: t.startTime && Date.now() - t.startTime || 0
 }
 } else if (e instanceof TouchEvent) {
 var v, y, m, g, b, w, A;
 n = {
 type: a.Touch,
 x: e.touches[0].clientX,
 y: e.touches[0].clientY,
 preX: (null === (v = t.preEvent) || void 0 === v ? void 0 : v.x) || 0,
 preY: (null === (y = t.preEvent) || void 0 === y ? void 0 : y.y) || 0,
 changeDistance: t.countDistance && (null === (m = t.preEvent) || void 0 === m ? void 0 : m.x) && (null === (g = t.preEvent) || void 0 === g ? void 0 : g.y) && o(e.touches[0].clientX, e.touches[0].clientY, null === (b = t.preEvent) || void 0 === b ? void 0 : b.x, null === (w = t.preEvent) || void 0 === w ? void 0 : w.y) || 0,
 timestamp: t.startTime && Date.now() - t.startTime || 0,
 preType: null === (A = t.preEvent) || void 0 === A ? void 0 : A.type
 }
 }
 n && (t.preEvent && n.timestamp > t.preEvent.timestamp ? t.logStack.push(n) : t.preEvent || t.logStack.push(n)),
 t.logStack.length > t.logStackMaxLength && t.logStack.shift(),
 t.preEvent = n
 }
 ))
 }
 }, {
 key: "start",
 value: function() {
 var t = this;
 i() ? console.error("ActivityDetector can not be used in SSR") : (this.startTime = Date.now(),
 this.state = "start",
 this.activityEvents.forEach((function(e) {
 window.addEventListener(e, t.logger)
 }
 )))
 }
 }, {
 key: "stop",
 value: function() {
 var t = this;
 i() ? console.error("ActivityDetector can not be used in SSR") : (this.startTime = void 0,
 this.state = "stop",
 this.activityEvents.forEach((function(e) {
 window.removeEventListener(e, t.logger)
 }
 )))
 }
 }, {
 key: "clearLog",
 value: function() {
 this.logStack = []
 }
 }, {
 key: "getPreTimeLog",
 value: function(t) {
 if (t) {
 var e, n = null === (e = this.logStack[this.logStack.length - 1]) || void 0 === e ? void 0 : e.timestamp;
 return n && n >= t ? this.logStack.filter((function(e) {
 return e.timestamp && e.timestamp >= n - t
 }
 )) : this.logStack
 }
 return this.logStack
 }
 }, {
 key: "getLog",
 value: function(t, e) {
 return t && e && t < e ? this.logStack.filter((function(n, r, o) {
 return !(r && n.timestamp < o[r - 1].timestamp) && n.timestamp && n.timestamp >= t && n.timestamp <= e
 }
 )) : !e && t ? this.logStack.filter((function(e, n, r) {
 return !(n && e.timestamp < r[n - 1].timestamp) && e.timestamp && e.timestamp >= t
 }
 )) : !t && e ? this.logStack.filter((function(t, n, r) {
 return !(n && t.timestamp < r[n - 1].timestamp) && t.timestamp && t.timestamp <= e
 }
 )) : this.logStack
 }
 }]

This userLog is collected from breakpoint debug:

"[{"type":"mousemove","x":317,"y":332,"preX":0,"preY":0,"changeDistance":0,"timestamp":43},{"type":"mousemove","x":317,"y":343,"preX":317,"preY":332,"changeDistance":0,"timestamp":82,"preType":"mousemove"},{"type":"mousemove","x":317,"y":387,"preX":317,"preY":343,"changeDistance":0,"timestamp":202,"preType":"mousemove"},{"type":"mousemove","x":317,"y":405,"preX":317,"preY":387,"changeDistance":0,"timestamp":335,"preType":"mousemove"}]"

So, these are some additional processes to dm_img_list.


I have found the transformation:

 function(t) {
 var e = N
 , n = function(t, e, n, r, o, i) {
 return t(e, n, r, o, i)
 }
 , r = t[e(519)]((function(t, e) {
 var r = t.x
 , o = void 0 === r ? 0 : r
 , i = t.y
 , a = void 0 === i ? 0 : i
 , c = t.timestamp
 , u = t.type;
 return n(B, o, a, c, u, e)
 }
 ))[e(519)]((function(t) {
 return {
 x: t[0],
 y: t[1],
 z: t[2],
 timestamp: t[3],
 type: t[4]
 }
 }
 ));
 try {
 return JSON.stringify(r)
 } catch (t) {
 return console[e(541)](t),
 ""
 }
 }

Transformation:

function transform(x, y, timestamp, type, index) {
 let c = Math.random() * 114 * index;
 return {
 x: 3 * x + 2 * y + c,
 y: 4 * x - 5 * y + c,
 z: c,
 timestamp: timestamp,
 type: TypeMap[type],
 };
}

TypeMap:

click: 1
focus: 5
keydown: 2
mousemove: 0
touch: 4
wheel: 3
However, the `dm_img_list` (mouse & keyboard actions record) has not been forged. This may be not working in next days. ------------------------- Sample of `dm_img_list`: ``` %7B%22x%22:302,%22y%22:395,%22z%22:0,%22timestamp%22:73,%22type%22:0%7D,%7B%22x%22:396,%22y%22:393,%22z%22:37,%22timestamp%22:192,%22type%22:0%7D,%7B%22x%22:724,%22y%22:491,%22z%22:158,%22timestamp%22:332,%22type%22:0%7D ``` URL decoded (Json Array without brackets): ``` {"x":302,"y":395,"z":0,"timestamp":73,"type":0},{"x":396,"y":393,"z":37,"timestamp":192,"type":0},{"x":724,"y":491,"z":158,"timestamp":332,"type":0} ``` ------------------------- `dm_img_list` is generated in `https://s1.hdslb.com/bfs/static/jinkela/space/9.space.85c18df037f33e9da25b31e5462b10906c75adfc.js` as **userLog**: function `queryUserLog`: ```javascript function(t) { var e, n, r = t.preTime, o = t.startTime, i = t.endTime; e = r ? this.getPreTimeLog(r) : this.getLog(o, i); try { n = JSON.stringify(e) } catch (t) { n = "[]" } return { userLog: e, userLogStr: n, userAgent: navigator.userAgent, webglStr: this.webglStr, webglVendorAndRenderer: this.webglVendorAndRenderer } ``` We can see this is accumulated. function `getLog`: ```javascript function(t, e) { return this.activityDetector.getLog(t, e) } ``` ----------------------------- And this may be desugared `activityDetector` class ```javascript [{ key: "initLogger", value: function() { var t = this; return Object(c.b)(this.samplingTime, (function(e) { var n, r, i, c, u, s, l, f; if (e instanceof MouseEvent) n = { type: e.type, x: e.clientX, y: e.clientY, preX: (null === (r = t.preEvent) || void 0 === r ? void 0 : r.x) || 0, preY: (null === (i = t.preEvent) || void 0 === i ? void 0 : i.y) || 0, changeDistance: t.countDistance && (null === (c = t.preEvent) || void 0 === c ? void 0 : c.x) && (null === (u = t.preEvent) || void 0 === u ? void 0 : u.y) && o(e.clientX, e.clientY, null === (s = t.preEvent) || void 0 === s ? void 0 : s.x, null === (l = t.preEvent) || void 0 === l ? void 0 : l.y) || 0, timestamp: t.startTime && Date.now() - t.startTime || 0, preType: null === (f = t.preEvent) || void 0 === f ? void 0 : f.type }, e instanceof WheelEvent && (n.deltaX = e.deltaX, n.deltaY = e.deltaY); else if (e instanceof KeyboardEvent) { var p, d; n = { type: a.Keydown, key: e.key, preKey: (null === (p = t.preEvent) || void 0 === p ? void 0 : p.key) || "mouse", preType: null === (d = t.preEvent) || void 0 === d ? void 0 : d.type, timestamp: t.startTime && Date.now() - t.startTime || 0 } } else if (e instanceof FocusEvent) { var h; n = { type: a.Focus, preType: null === (h = t.preEvent) || void 0 === h ? void 0 : h.type, timestamp: t.startTime && Date.now() - t.startTime || 0 } } else if (e instanceof TouchEvent) { var v, y, m, g, b, w, A; n = { type: a.Touch, x: e.touches[0].clientX, y: e.touches[0].clientY, preX: (null === (v = t.preEvent) || void 0 === v ? void 0 : v.x) || 0, preY: (null === (y = t.preEvent) || void 0 === y ? void 0 : y.y) || 0, changeDistance: t.countDistance && (null === (m = t.preEvent) || void 0 === m ? void 0 : m.x) && (null === (g = t.preEvent) || void 0 === g ? void 0 : g.y) && o(e.touches[0].clientX, e.touches[0].clientY, null === (b = t.preEvent) || void 0 === b ? void 0 : b.x, null === (w = t.preEvent) || void 0 === w ? void 0 : w.y) || 0, timestamp: t.startTime && Date.now() - t.startTime || 0, preType: null === (A = t.preEvent) || void 0 === A ? void 0 : A.type } } n && (t.preEvent && n.timestamp > t.preEvent.timestamp ? t.logStack.push(n) : t.preEvent || t.logStack.push(n)), t.logStack.length > t.logStackMaxLength && t.logStack.shift(), t.preEvent = n } )) } }, { key: "start", value: function() { var t = this; i() ? console.error("ActivityDetector can not be used in SSR") : (this.startTime = Date.now(), this.state = "start", this.activityEvents.forEach((function(e) { window.addEventListener(e, t.logger) } ))) } }, { key: "stop", value: function() { var t = this; i() ? console.error("ActivityDetector can not be used in SSR") : (this.startTime = void 0, this.state = "stop", this.activityEvents.forEach((function(e) { window.removeEventListener(e, t.logger) } ))) } }, { key: "clearLog", value: function() { this.logStack = [] } }, { key: "getPreTimeLog", value: function(t) { if (t) { var e, n = null === (e = this.logStack[this.logStack.length - 1]) || void 0 === e ? void 0 : e.timestamp; return n && n >= t ? this.logStack.filter((function(e) { return e.timestamp && e.timestamp >= n - t } )) : this.logStack } return this.logStack } }, { key: "getLog", value: function(t, e) { return t && e && t < e ? this.logStack.filter((function(n, r, o) { return !(r && n.timestamp < o[r - 1].timestamp) && n.timestamp && n.timestamp >= t && n.timestamp <= e } )) : !e && t ? this.logStack.filter((function(e, n, r) { return !(n && e.timestamp < r[n - 1].timestamp) && e.timestamp && e.timestamp >= t } )) : !t && e ? this.logStack.filter((function(t, n, r) { return !(n && t.timestamp < r[n - 1].timestamp) && t.timestamp && t.timestamp <= e } )) : this.logStack } }] ``` ------------------------------ This userLog is collected from breakpoint debug: ``` "[{"type":"mousemove","x":317,"y":332,"preX":0,"preY":0,"changeDistance":0,"timestamp":43},{"type":"mousemove","x":317,"y":343,"preX":317,"preY":332,"changeDistance":0,"timestamp":82,"preType":"mousemove"},{"type":"mousemove","x":317,"y":387,"preX":317,"preY":343,"changeDistance":0,"timestamp":202,"preType":"mousemove"},{"type":"mousemove","x":317,"y":405,"preX":317,"preY":387,"changeDistance":0,"timestamp":335,"preType":"mousemove"}]" ``` So, these are some additional processes to `dm_img_list`. ------------------------------------ I have found the transformation: ``` javascript function(t) { var e = N , n = function(t, e, n, r, o, i) { return t(e, n, r, o, i) } , r = t[e(519)]((function(t, e) { var r = t.x , o = void 0 === r ? 0 : r , i = t.y , a = void 0 === i ? 0 : i , c = t.timestamp , u = t.type; return n(B, o, a, c, u, e) } ))[e(519)]((function(t) { return { x: t[0], y: t[1], z: t[2], timestamp: t[3], type: t[4] } } )); try { return JSON.stringify(r) } catch (t) { return console[e(541)](t), "" } } ``` ----------------------------------- Transformation: ``` function transform(x, y, timestamp, type, index) { let c = Math.random() * 114 * index; return { x: 3 * x + 2 * y + c, y: 4 * x - 5 * y + c, z: c, timestamp: timestamp, type: TypeMap[type], }; } ``` TypeMap: ``` click: 1 focus: 5 keydown: 2 mousemove: 0 touch: 4 wheel: 3 ```
Author
Contributor
Copy link

Actually, I met some "reCaptcha" challenges on web.

So, if we can handle challenge, so we can bypass anyway!

Actually, I met some "reCaptcha" challenges on web. So, if we can handle challenge, so we can bypass anyway!
Author
Contributor
Copy link
function transform(positionX, positionY, timestamp, type, index) {
 let TypeMap = {
 mousemove: 0,
 click: 1,
 keydown: 2,
 wheel: 3,
 touch: 4,
 focus: 5,
 };
 let c = Math.random() * 114 * index;
 return {
 x: 3 * positionX + 2 * positionY + c,
 y: 4 * positionX - 5 * positionY + c,
 z: c,
 timestamp: timestamp,
 type: TypeMap[type],
 };
}

This is how items in dm_img_list generated.

```javascript function transform(positionX, positionY, timestamp, type, index) { let TypeMap = { mousemove: 0, click: 1, keydown: 2, wheel: 3, touch: 4, focus: 5, }; let c = Math.random() * 114 * index; return { x: 3 * positionX + 2 * positionY + c, y: 4 * positionX - 5 * positionY + c, z: c, timestamp: timestamp, type: TypeMap[type], }; } ``` This is how items in `dm_img_list` generated.

Thank you! Works great.

Thank you! Works great.
Sign in to join this conversation.
No reviewers
Labels
Clear labels
No items
No labels
Milestone
Clear milestone
No items
No milestone
Projects
Clear projects
No items
No project
Assignees
Clear assignees
No assignees
2 participants
Notifications
Due date
The due date is invalid or out of range. Please use the format "yyyy-mm-dd".

No due date set.

Dependencies

No dependencies set.

Reference
NullPointerException/PipePipeExtractor!3
Reference in a new issue
NullPointerException/PipePipeExtractor
No description provided.
Delete branch "chr_56/PipePipeExtractor:random_device"

Deleting a branch is permanent. Although the deleted branch may continue to exist for a short time before it actually gets removed, it CANNOT be undone in most cases. Continue?