Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit e319736

Browse files
author
hulutter
committed
feat(privateNpmRegistry): enable package serving via internal registry proxy and package cache
1 parent 42c616b commit e319736

File tree

5 files changed

+323
-4
lines changed

5 files changed

+323
-4
lines changed

‎client/packages/lowcoder/src/comps/utils/remote.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export function getRemoteCompType(
1212
}
1313

1414
export function parseCompType(compType: string) {
15-
const [type, source, packageNameAndVersion, compName] = compType.split("#");
15+
let [type, source, packageNameAndVersion, compName] = compType.split("#");
1616
const isRemote = type === "remote";
1717

1818
if (!isRemote) {
@@ -22,7 +22,13 @@ export function parseCompType(compType: string) {
2222
};
2323
}
2424

25-
const [packageName, packageVersion] = packageNameAndVersion.split("@");
25+
const packageRegex = /^(?<packageName>(@[a-z0-9-~][a-z0-9-._~]*\/)?[a-z0-9-~][a-z0-9-._~]*)@(?<packageVersion>([0-9]+.[0-9]+.[0-9]+)(-[\w\d-]+)?)$/;
26+
const matches = packageNameAndVersion.match(packageRegex);
27+
if (!matches?.groups) {
28+
throw new Error(`Invalid package name and version: ${packageNameAndVersion}`);
29+
}
30+
31+
const {packageName, packageVersion} = matches.groups;
2632
return {
2733
compName,
2834
isRemote,
Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
1-
export const NPM_REGISTRY_URL = "https://registry.npmjs.com";
2-
export const NPM_PLUGIN_ASSETS_BASE_URL = "https://unpkg.com";
1+
import { sdkConfig } from "./sdkConfig";
2+
3+
const baseUrl = sdkConfig.baseURL || LOWCODER_NODE_SERVICE_URL || "";
4+
export const NPM_REGISTRY_URL = `${baseUrl}/node-service/api/npm/registry`;
5+
export const NPM_PLUGIN_ASSETS_BASE_URL = `${baseUrl}/node-service/api/npm/package`;
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
import "../common/logger";
2+
import fs from "fs/promises";
3+
import { spawn } from "child_process";
4+
import { Request as ServerRequest, Response as ServerResponse } from "express";
5+
import { NpmRegistryService, NpmRegistryConfigEntry } from "../services/npmRegistry";
6+
7+
8+
type PackagesVersionInfo = {
9+
"dist-tags": {
10+
latest: string
11+
},
12+
versions: {
13+
[version: string]: {
14+
dist: {
15+
tarball: string
16+
}
17+
}
18+
}
19+
};
20+
21+
22+
/**
23+
* Initializes npm registry cache directory
24+
*/
25+
const CACHE_DIR = process.env.NPM_CACHE_DIR || "/tmp/npm-package-cache";
26+
try {
27+
fs.mkdir(CACHE_DIR, { recursive: true });
28+
} catch (error) {
29+
console.error("Error creating cache directory", error);
30+
}
31+
32+
33+
/**
34+
* Fetches package info from npm registry
35+
*/
36+
const fetchRegistryBasePath = "/npm/registry";
37+
export async function fetchRegistry(request: ServerRequest, response: ServerResponse) {
38+
try {
39+
const path = request.path.replace(fetchRegistryBasePath, "");
40+
logger.info(`Fetch registry info for path: ${path}`);
41+
42+
const pathPackageInfo = parsePackageInfoFromPath(path);
43+
if (!pathPackageInfo) {
44+
return response.status(400).send(`Invalid package path: ${path}`);
45+
}
46+
const {organization, name} = pathPackageInfo;
47+
const packageName = organization ? `@${organization}/${name}` : name;
48+
49+
const registryResponse = await fetchFromRegistry(packageName, path);
50+
response.json(await registryResponse.json());
51+
} catch (error) {
52+
logger.error("Error fetching registry", error);
53+
response.status(500).send("Internal server error");
54+
}
55+
}
56+
57+
58+
/**
59+
* Fetches package files from npm registry if not yet cached
60+
*/
61+
const fetchPackageFileBasePath = "/npm/package";
62+
export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) {
63+
try {
64+
const path = request.path.replace(fetchPackageFileBasePath, "");
65+
logger.info(`Fetch file for path: ${path}`);
66+
67+
const pathPackageInfo = parsePackageInfoFromPath(path);
68+
if (!pathPackageInfo) {
69+
return response.status(400).send(`Invalid package path: ${path}`);
70+
}
71+
72+
logger.info(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`);
73+
const {organization, name, version, file} = pathPackageInfo;
74+
const packageName = organization ? `@${organization}/${name}` : name;
75+
let packageVersion = version;
76+
77+
let packageInfo: PackagesVersionInfo | null = null;
78+
if (version === "latest") {
79+
const packageInfo: PackagesVersionInfo = await fetchPackageInfo(packageName);
80+
packageVersion = packageInfo["dist-tags"].latest;
81+
}
82+
83+
const packageBaseDir = `${CACHE_DIR}/${packageName}/${packageVersion}/package`;
84+
const packageExists = await fileExists(`${packageBaseDir}/package.json`)
85+
if (!packageExists) {
86+
if (!packageInfo) {
87+
packageInfo = await fetchPackageInfo(packageName);
88+
}
89+
90+
if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) {
91+
return response.status(404).send("Not found");
92+
}
93+
94+
const tarball = packageInfo.versions[packageVersion].dist.tarball;
95+
logger.info("Fetching tarball...", tarball);
96+
await fetchAndUnpackTarball(tarball, packageName, packageVersion);
97+
}
98+
99+
// Fallback to index.mjs if index.js is not present
100+
if (file === "index.js" && !await fileExists(`${packageBaseDir}/${file}`)) {
101+
logger.info("Fallback to index.mjs");
102+
return response.sendFile(`${packageBaseDir}/index.mjs`);
103+
}
104+
105+
return response.sendFile(`${packageBaseDir}/${file}`);
106+
} catch (error) {
107+
logger.error("Error fetching package file", error);
108+
response.status(500).send("Internal server error");
109+
}
110+
};
111+
112+
113+
/**
114+
* Helpers
115+
*/
116+
117+
function parsePackageInfoFromPath(path: string): {organization: string, name: string, version: string, file: string} | undefined {
118+
logger.info(`Parse package info from path: ${path}`);
119+
//@ts-ignore - regex groups
120+
const packageInfoRegex = /^\/?(?<fullName>(?:@(?<organization>[a-z0-9-~][a-z0-9-._~]*)\/)?(?<name>[a-z0-9-~][a-z0-9-._~]*))(?:@(?<version>[-a-z0-9><=_.^~]+))?\/(?<file>[^\r\n]*)?$/;
121+
const matches = path.match(packageInfoRegex);
122+
logger.info(`Parse package matches: ${JSON.stringify(matches)}`);
123+
if (!matches?.groups) {
124+
return;
125+
}
126+
127+
let {organization, name, version, file} = matches.groups;
128+
version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest";
129+
130+
return {organization, name, version, file};
131+
}
132+
133+
function fetchFromRegistry(packageName: string, urlOrPath: string): Promise<Response> {
134+
const config: NpmRegistryConfigEntry = NpmRegistryService.getInstance().getRegistryEntryForPackage(packageName);
135+
const registryUrl = config?.registry.url;
136+
137+
const headers: {[key: string]: string} = {};
138+
switch (config?.registry.auth.type) {
139+
case "none":
140+
break;
141+
case "basic":
142+
const basicUserPass = config?.registry.auth?.credentials;
143+
headers["Authorization"] = `Basic ${basicUserPass}`;
144+
break;
145+
case "bearer":
146+
const bearerToken = config?.registry.auth?.credentials;
147+
headers["Authorization"] = `Bearer ${bearerToken}`;
148+
break;
149+
}
150+
151+
let url = urlOrPath;
152+
if (!urlOrPath.startsWith("http")) {
153+
const separator = urlOrPath.startsWith("/") ? "" : "/";
154+
url = `${registryUrl}${separator}${urlOrPath}`;
155+
}
156+
157+
logger.debug(`Fetch from registry: ${url}`);
158+
return fetch(url, {headers});
159+
}
160+
161+
function fetchPackageInfo(packageName: string): Promise<PackagesVersionInfo> {
162+
return fetchFromRegistry(packageName, packageName).then(res => res.json());
163+
}
164+
165+
async function fetchAndUnpackTarball(url: string, packageName: string, packageVersion: string) {
166+
const response: Response = await fetchFromRegistry(packageName, url);
167+
const arrayBuffer = await response.arrayBuffer();
168+
const buffer = Buffer.from(arrayBuffer);
169+
const path = `${CACHE_DIR}/${url.split("/").pop()}`;
170+
await fs.writeFile(path, buffer);
171+
await unpackTarball(path, packageName, packageVersion);
172+
await fs.unlink(path);
173+
}
174+
175+
async function unpackTarball(path: string, packageName: string, packageVersion: string) {
176+
const destinationPath = `${CACHE_DIR}/${packageName}/${packageVersion}`;
177+
await fs.mkdir(destinationPath, { recursive: true });
178+
await new Promise<void> ((resolve, reject) => {
179+
const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]);
180+
tar.stdout.on("data", (data) => logger.info(data));
181+
tar.stderr.on("data", (data) => console.error(data));
182+
tar.on("close", (code) => {
183+
code === 0 ? resolve() : reject();
184+
});
185+
});
186+
}
187+
188+
async function fileExists(filePath: string): Promise<boolean> {
189+
try {
190+
await fs.access(filePath);
191+
return true;
192+
} catch (error) {
193+
return false;
194+
}
195+
}

‎server/node-service/src/routes/apiRouter.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import express from "express";
22
import * as pluginControllers from "../controllers/plugins";
33
import jsControllers from "../controllers/runJavascript";
4+
import * as npmControllers from "../controllers/npm";
45

56
const apiRouter = express.Router();
67

@@ -12,4 +13,7 @@ apiRouter.post("/runPluginQuery", pluginControllers.runPluginQuery);
1213
apiRouter.post("/getPluginDynamicConfig", pluginControllers.getDynamicDef);
1314
apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePluginDataSourceConfig);
1415

16+
apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry);
17+
apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile);
18+
1519
export default apiRouter;
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
type BasicAuthType = {
2+
type: "basic",
3+
credentials: string,
4+
}
5+
6+
type BearerAuthType = {
7+
type: "bearer",
8+
credentials: string,
9+
};
10+
11+
type NoAuthType = {
12+
type: "none"
13+
};
14+
15+
type OrganizationScope = {
16+
type: "organization",
17+
pattern: string
18+
};
19+
20+
type PackageScope = {
21+
type: "package",
22+
pattern: string
23+
};
24+
25+
type GlobalScope = {
26+
type: "global"
27+
};
28+
29+
export type NpmRegistryConfigEntry = {
30+
scope: OrganizationScope | PackageScope | GlobalScope,
31+
registry: {
32+
url: string,
33+
auth: BasicAuthType | BearerAuthType | NoAuthType
34+
}
35+
};
36+
37+
export type NpmRegistryConfig = NpmRegistryConfigEntry[];
38+
39+
export class NpmRegistryService {
40+
41+
public static DEFAULT_REGISTRY: NpmRegistryConfigEntry = {
42+
scope: { type: "global" },
43+
registry: {
44+
url: "https://registry.npmjs.org",
45+
auth: { type: "none" }
46+
}
47+
};
48+
49+
private static instance: NpmRegistryService;
50+
51+
private readonly registryConfig: NpmRegistryConfig = [];
52+
53+
private constructor() {
54+
const registryConfig = this.getRegistryConfig();
55+
if (registryConfig.length === 0 || !registryConfig.some(entry => entry.scope.type === "global")) {
56+
registryConfig.push(NpmRegistryService.DEFAULT_REGISTRY);
57+
}
58+
this.registryConfig = registryConfig;
59+
}
60+
61+
public static getInstance(): NpmRegistryService {
62+
if (!NpmRegistryService.instance) {
63+
NpmRegistryService.instance = new NpmRegistryService();
64+
}
65+
return NpmRegistryService.instance;
66+
}
67+
68+
private getRegistryConfig(): NpmRegistryConfig {
69+
const registryConfig = process.env.NPM_REGISTRY_CONFIG;
70+
if (!registryConfig) {
71+
return [];
72+
}
73+
74+
try {
75+
const config = JSON.parse(registryConfig);
76+
return NpmRegistryService.sortRegistryConfig(config);
77+
} catch (error) {
78+
console.error("Error parsing registry config", error);
79+
return [];
80+
}
81+
}
82+
83+
private static sortRegistryConfig(registryConfig: NpmRegistryConfig): NpmRegistryConfig {
84+
const globalRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "global");
85+
const orgRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "organization");
86+
const packageRegistries = registryConfig.filter((entry: NpmRegistryConfigEntry) => entry.scope.type === "package");
87+
// Order of precedence: package > organization > global
88+
return [...packageRegistries, ...orgRegistries, ...globalRegistries];
89+
}
90+
91+
public getRegistryEntryForPackage(packageName: string): NpmRegistryConfigEntry {
92+
const config: NpmRegistryConfigEntry | undefined = this.registryConfig.find(entry => {
93+
if (entry.scope.type === "organization") {
94+
return packageName.startsWith(entry.scope.pattern);
95+
} else if (entry.scope.type === "package") {
96+
return packageName === entry.scope.pattern;
97+
} else {
98+
return true;
99+
}
100+
});
101+
102+
if (!config) {
103+
logger.info(`No registry entry found for package: ${packageName}`);
104+
return NpmRegistryService.DEFAULT_REGISTRY;
105+
} else {
106+
logger.info(`Found registry entry for package: ${packageName} -> ${config.registry.url}`);
107+
}
108+
109+
return config;
110+
}
111+
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /