Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 4917aef

Browse files
fubhyclaude
andcommitted
node, chain: Add extensible compression support for RPC requests
- Replace boolean compression_enabled with Compression enum (None, Gzip) - Support per-provider compression configuration via "compression" field - Add placeholders for future compression methods (Brotli, Deflate) - Update transport layer to handle compression enum with match statement - Add comprehensive unit tests for compression configuration parsing - Update example configuration and documentation Configuration examples: compression = "gzip" # Enable gzip compression compression = "none" # Disable compression (default) Addresses issue #5671 with future-extensible design. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent d4ddfaf commit 4917aef

File tree

7 files changed

+225
-6
lines changed

7 files changed

+225
-6
lines changed

‎Cargo.lock

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎chain/ethereum/src/transport.rs

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use graph::components::network_provider::ProviderName;
2+
use graph::config::Compression;
23
use graph::endpoint::{EndpointMetrics, RequestLabels};
34
use jsonrpc_core::types::Call;
45
use jsonrpc_core::Value;
@@ -54,12 +55,25 @@ impl Transport {
5455
headers: graph::http::HeaderMap,
5556
metrics: Arc<EndpointMetrics>,
5657
provider: impl AsRef<str>,
58+
compression: Compression,
5759
) -> Self {
5860
// Unwrap: This only fails if something is wrong with the system's TLS config.
59-
let client = reqwest::Client::builder()
60-
.default_headers(headers)
61-
.build()
62-
.unwrap();
61+
let mut client_builder = reqwest::Client::builder().default_headers(headers);
62+
63+
match compression {
64+
Compression::Gzip => {
65+
// Enable gzip compression/decompression for requests and responses
66+
client_builder = client_builder.gzip(true);
67+
}
68+
Compression::None => {
69+
// No compression
70+
} // Future compression methods can be handled here:
71+
// Compression::Brotli => {
72+
// client_builder = client_builder.brotli(true);
73+
// }
74+
}
75+
76+
let client = client_builder.build().unwrap();
6377

6478
Transport::RPC {
6579
client: http::Http::with_client(client, rpc),

‎graph/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ chrono = "0.4.41"
2626
envconfig = "0.11.0"
2727
Inflector = "0.11.3"
2828
atty = "0.2"
29-
reqwest = { version = "0.12.15", features = ["json", "stream", "multipart"] }
29+
reqwest = { version = "0.12.15", features = ["json", "stream", "multipart", "gzip"] }
3030
ethabi = "17.2"
3131
hex = "0.4.3"
3232
http0 = { version = "0", package = "http" }
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Plan: Implement Extensible Compression for RPC Requests
2+
3+
## Overview
4+
Add extensible compression support for Graph Node's outgoing RPC requests to upstream providers, configurable on a per-provider basis with future compression methods in mind.
5+
6+
## Implementation Steps (COMPLETED)
7+
8+
### 1. ✅ Create Compression Enum (`node/src/config.rs`)
9+
- Added `Compression` enum with `None` and `Gzip` variants
10+
- Commented placeholders for future compression methods (Brotli, Deflate)
11+
- Default implementation returns `Compression::None`
12+
13+
### 2. ✅ Update Configuration Structure (`node/src/config.rs`)
14+
- Replaced `compression_enabled: bool` with `compression: Compression` field in `Web3Provider` struct
15+
- Updated all existing code to use new enum
16+
- Added unit tests for both "gzip" and "none" compression options
17+
18+
### 3. ✅ Modify HTTP Transport (`chain/ethereum/src/transport.rs`)
19+
- Updated `Transport::new_rpc()` to accept `Compression` enum parameter
20+
- Implemented match statement for different compression types
21+
- Added comments showing where future compression methods can be added
22+
- Uses reqwest's `.gzip(true)` for automatic compression/decompression
23+
24+
### 4. ✅ Update Transport Creation (`node/src/chain.rs`)
25+
- Pass compression enum from config to transport
26+
- Updated logging to show compression method using debug format
27+
28+
### 5. ✅ Update Dependencies (`graph/Cargo.toml`)
29+
- Added "gzip" feature to reqwest dependency
30+
31+
### 6. ✅ Update Test Configuration
32+
- Updated `full_config.toml` example to use new enum format
33+
- Added comprehensive unit tests for compression parsing
34+
35+
## Configuration Examples
36+
37+
### Gzip Compression
38+
```toml
39+
[chains.mainnet]
40+
provider = [
41+
{
42+
label = "mainnet-rpc",
43+
details = {
44+
type = "web3",
45+
url = "http://rpc.example.com",
46+
features = ["archive"],
47+
compression = "gzip"
48+
}
49+
}
50+
]
51+
```
52+
53+
### No Compression (Default)
54+
```toml
55+
[chains.mainnet]
56+
provider = [
57+
{
58+
label = "mainnet-rpc",
59+
details = {
60+
type = "web3",
61+
url = "http://rpc.example.com",
62+
features = ["archive"],
63+
compression = "none" # or omit entirely
64+
}
65+
}
66+
]
67+
```
68+
69+
### Future Extension Example
70+
```rust
71+
// Future compression methods can be easily added:
72+
#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq)]
73+
pub enum Compression {
74+
#[serde(rename = "none")]
75+
None,
76+
#[serde(rename = "gzip")]
77+
Gzip,
78+
#[serde(rename = "brotli")]
79+
Brotli,
80+
#[serde(rename = "deflate")]
81+
Deflate,
82+
}
83+
84+
// And handled in transport:
85+
match compression {
86+
Compression::Gzip => client_builder = client_builder.gzip(true),
87+
Compression::Brotli => client_builder = client_builder.brotli(true),
88+
Compression::Deflate => client_builder = client_builder.deflate(true),
89+
Compression::None => {} // No compression
90+
}
91+
```
92+
93+
## Benefits of This Implementation
94+
- **Extensible**: Easy to add new compression methods without breaking changes
95+
- **Backward Compatible**: Defaults to no compression, existing configs work unchanged
96+
- **Type Safe**: Enum prevents invalid compression method strings
97+
- **Future Proof**: Clear pattern for adding Brotli, Deflate, etc.
98+
- **Per-Provider**: Each RPC provider can have different compression settings

‎node/resources/tests/full_config.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ shard = "primary"
4848
provider = [
4949
{ label = "mainnet-0", url = "http://rpc.mainnet.io", features = ["archive", "traces"] },
5050
{ label = "mainnet-1", details = { type = "web3call", url = "http://rpc.mainnet.io", features = ["archive", "traces"] }},
51+
{ label = "mainnet-2", details = { type = "web3", url = "http://rpc.mainnet.io", features = ["archive"], compression = "gzip" }},
5152
{ label = "firehose", details = { type = "firehose", url = "http://localhost:9000", features = [] }},
5253
{ label = "substreams", details = { type = "substreams", url = "http://localhost:9000", features = [] }},
5354
]

‎node/src/chain.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,8 @@ pub async fn create_ethereum_networks_for_chain(
282282
logger,
283283
"Creating transport";
284284
"url" => &web3.url,
285-
"capabilities" => capabilities
285+
"capabilities" => capabilities,
286+
"compression" => ?web3.compression
286287
);
287288

288289
use crate::config::Transport::*;
@@ -293,6 +294,7 @@ pub async fn create_ethereum_networks_for_chain(
293294
web3.headers.clone(),
294295
endpoint_metrics.cheap_clone(),
295296
&provider.label,
297+
web3.compression,
296298
),
297299
Ipc => Transport::new_ipc(&web3.url).await,
298300
Ws => Transport::new_ws(&web3.url).await,

‎node/src/config.rs

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,7 @@ impl ChainSection {
502502
features,
503503
headers: Default::default(),
504504
rules: vec![],
505+
compression: Compression::None,
505506
}),
506507
};
507508
let entry = chains.entry(name.to_string()).or_insert_with(|| Chain {
@@ -705,6 +706,10 @@ pub struct Web3Provider {
705706

706707
#[serde(default, rename = "match")]
707708
rules: Vec<Web3Rule>,
709+
710+
/// Compression method for RPC requests and responses
711+
#[serde(default)]
712+
pub compression: Compression,
708713
}
709714

710715
impl Web3Provider {
@@ -901,6 +906,7 @@ impl<'de> Deserialize<'de> for Provider {
901906
.ok_or_else(|| serde::de::Error::missing_field("features"))?,
902907
headers: headers.unwrap_or_else(HeaderMap::new),
903908
rules: nodes,
909+
compression: Compression::None,
904910
}),
905911
};
906912

@@ -944,6 +950,25 @@ pub enum Transport {
944950
Ipc,
945951
}
946952

953+
#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq)]
954+
pub enum Compression {
955+
#[serde(rename = "none")]
956+
None,
957+
#[serde(rename = "gzip")]
958+
Gzip,
959+
// Future compression methods can be added here:
960+
// #[serde(rename = "brotli")]
961+
// Brotli,
962+
// #[serde(rename = "deflate")]
963+
// Deflate,
964+
}
965+
966+
impl Default for Compression {
967+
fn default() -> Self {
968+
Compression::None
969+
}
970+
}
971+
947972
impl Default for Transport {
948973
fn default() -> Self {
949974
Self::Rpc
@@ -1307,6 +1332,7 @@ mod tests {
13071332
features: BTreeSet::new(),
13081333
headers: HeaderMap::new(),
13091334
rules: Vec::new(),
1335+
compression: Compression::None,
13101336
}),
13111337
},
13121338
actual
@@ -1333,6 +1359,7 @@ mod tests {
13331359
features: BTreeSet::new(),
13341360
headers: HeaderMap::new(),
13351361
rules: Vec::new(),
1362+
compression: Compression::None,
13361363
}),
13371364
},
13381365
actual
@@ -1440,6 +1467,7 @@ mod tests {
14401467
features,
14411468
headers,
14421469
rules: Vec::new(),
1470+
compression: Compression::None,
14431471
}),
14441472
},
14451473
actual
@@ -1465,6 +1493,7 @@ mod tests {
14651493
features: BTreeSet::new(),
14661494
headers: HeaderMap::new(),
14671495
rules: Vec::new(),
1496+
compression: Compression::None,
14681497
}),
14691498
},
14701499
actual
@@ -1834,6 +1863,7 @@ mod tests {
18341863
features: BTreeSet::new(),
18351864
headers: HeaderMap::new(),
18361865
rules: Vec::new(),
1866+
compression: Compression::None,
18371867
}),
18381868
},
18391869
actual
@@ -1846,6 +1876,66 @@ mod tests {
18461876
assert!(SubgraphLimit::Limit(10) > SubgraphLimit::Disabled);
18471877
}
18481878

1879+
#[test]
1880+
fn it_parses_web3_provider_with_compression() {
1881+
let actual = toml::from_str(
1882+
r#"
1883+
label = "compressed"
1884+
details = { type = "web3", url = "http://localhost:8545", features = ["archive"], compression = "gzip" }
1885+
"#,
1886+
)
1887+
.unwrap();
1888+
1889+
assert_eq!(
1890+
Provider {
1891+
label: "compressed".to_owned(),
1892+
details: ProviderDetails::Web3(Web3Provider {
1893+
transport: Transport::Rpc,
1894+
url: "http://localhost:8545".to_owned(),
1895+
features: {
1896+
let mut features = BTreeSet::new();
1897+
features.insert("archive".to_string());
1898+
features
1899+
},
1900+
headers: HeaderMap::new(),
1901+
rules: Vec::new(),
1902+
compression: Compression::Gzip,
1903+
}),
1904+
},
1905+
actual
1906+
);
1907+
}
1908+
1909+
#[test]
1910+
fn it_parses_web3_provider_with_no_compression() {
1911+
let actual = toml::from_str(
1912+
r#"
1913+
label = "uncompressed"
1914+
details = { type = "web3", url = "http://localhost:8545", features = ["archive"], compression = "none" }
1915+
"#,
1916+
)
1917+
.unwrap();
1918+
1919+
assert_eq!(
1920+
Provider {
1921+
label: "uncompressed".to_owned(),
1922+
details: ProviderDetails::Web3(Web3Provider {
1923+
transport: Transport::Rpc,
1924+
url: "http://localhost:8545".to_owned(),
1925+
features: {
1926+
let mut features = BTreeSet::new();
1927+
features.insert("archive".to_string());
1928+
features
1929+
},
1930+
headers: HeaderMap::new(),
1931+
rules: Vec::new(),
1932+
compression: Compression::None,
1933+
}),
1934+
},
1935+
actual
1936+
);
1937+
}
1938+
18491939
#[test]
18501940
fn duplicated_labels_are_not_allowed_within_chain() {
18511941
let mut actual = toml::from_str::<ChainSection>(

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /