Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 89c0d38

Browse files
author
hulutter
committed
feat(privateNpmRegistry): add endpoints to fetch with config send via body and queue tar retrieval
1 parent e319736 commit 89c0d38

File tree

3 files changed

+171
-45
lines changed

3 files changed

+171
-45
lines changed

‎server/node-service/src/controllers/npm.ts‎

Lines changed: 146 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import "../common/logger";
22
import fs from "fs/promises";
33
import { spawn } from "child_process";
4-
import { Request as ServerRequest, Response as ServerResponse } from "express";
5-
import { NpmRegistryService, NpmRegistryConfigEntry } from "../services/npmRegistry";
4+
import { response,Request as ServerRequest, Response as ServerResponse } from "express";
5+
import { NpmRegistryService, NpmRegistryConfigEntry,NpmRegistryConfig } from "../services/npmRegistry";
66

77

88
type PackagesVersionInfo = {
@@ -19,21 +19,76 @@ type PackagesVersionInfo = {
1919
};
2020

2121

22+
class PackageProcessingQueue {
23+
public static readonly promiseRegistry: {[packageId: string]: Promise<void>} = {};
24+
public static readonly resolveRegistry: {[packageId: string]:() => void} = {};
25+
26+
public static add(packageId: string) {
27+
PackageProcessingQueue.promiseRegistry[packageId] = new Promise<void>((resolve) => {
28+
PackageProcessingQueue.resolveRegistry[packageId] = resolve;
29+
});
30+
}
31+
32+
public static has(packageId: string) {
33+
return !!PackageProcessingQueue.promiseRegistry[packageId];
34+
}
35+
36+
public static wait(packageId: string) {
37+
if (!PackageProcessingQueue.has(packageId)) {
38+
return Promise.resolve();
39+
}
40+
return PackageProcessingQueue.promiseRegistry[packageId];
41+
}
42+
43+
public static resolve(packageId: string) {
44+
if (!PackageProcessingQueue.has(packageId)) {
45+
return;
46+
}
47+
PackageProcessingQueue.resolveRegistry[packageId]();
48+
delete PackageProcessingQueue.promiseRegistry[packageId];
49+
delete PackageProcessingQueue.resolveRegistry[packageId];
50+
}
51+
}
52+
53+
2254
/**
2355
* Initializes npm registry cache directory
2456
*/
2557
const CACHE_DIR = process.env.NPM_CACHE_DIR || "/tmp/npm-package-cache";
2658
try {
2759
fs.mkdir(CACHE_DIR, { recursive: true });
2860
} catch (error) {
29-
console.error("Error creating cache directory", error);
61+
logger.error("Error creating cache directory", error);
3062
}
3163

3264

3365
/**
3466
* Fetches package info from npm registry
3567
*/
68+
3669
const fetchRegistryBasePath = "/npm/registry";
70+
71+
export async function fetchRegistryWithConfig(request: ServerRequest, response: ServerResponse) {
72+
try {
73+
const path = request.path.replace(fetchRegistryBasePath, "");
74+
logger.info(`Fetch registry info for path: ${path}`);
75+
76+
const pathPackageInfo = parsePackageInfoFromPath(path);
77+
if (!pathPackageInfo) {
78+
return response.status(400).send(`Invalid package path: ${path}`);
79+
}
80+
81+
const registryConfig: NpmRegistryConfig = request.body;
82+
const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig);
83+
84+
const registryResponse = await fetchFromRegistry(path, config);
85+
response.json(await registryResponse.json());
86+
} catch (error) {
87+
logger.error("Error fetching registry", error);
88+
response.status(500).send("Internal server error");
89+
}
90+
}
91+
3792
export async function fetchRegistry(request: ServerRequest, response: ServerResponse) {
3893
try {
3994
const path = request.path.replace(fetchRegistryBasePath, "");
@@ -43,10 +98,9 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp
4398
if (!pathPackageInfo) {
4499
return response.status(400).send(`Invalid package path: ${path}`);
45100
}
46-
const {organization, name} = pathPackageInfo;
47-
const packageName = organization ? `@${organization}/${name}` : name;
48101

49-
const registryResponse = await fetchFromRegistry(packageName, path);
102+
const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId);
103+
const registryResponse = await fetchFromRegistry(path, config);
50104
response.json(await registryResponse.json());
51105
} catch (error) {
52106
logger.error("Error fetching registry", error);
@@ -58,53 +112,100 @@ export async function fetchRegistry(request: ServerRequest, response: ServerResp
58112
/**
59113
* Fetches package files from npm registry if not yet cached
60114
*/
115+
61116
const fetchPackageFileBasePath = "/npm/package";
117+
118+
export async function fetchPackageFileWithConfig(request: ServerRequest, response: ServerResponse) {
119+
const path = request.path.replace(fetchPackageFileBasePath, "");
120+
logger.info(`Fetch file for path with config: ${path}`);
121+
122+
const pathPackageInfo = parsePackageInfoFromPath(path);
123+
if (!pathPackageInfo) {
124+
return response.status(400).send(`Invalid package path: ${path}`);
125+
}
126+
127+
const registryConfig: NpmRegistryConfig = request.body;
128+
const config = NpmRegistryService.getRegistryEntryForPackageWithConfig(pathPackageInfo.packageId, registryConfig);
129+
130+
fetchPackageFileInner(request, response, config);
131+
}
132+
62133
export async function fetchPackageFile(request: ServerRequest, response: ServerResponse) {
134+
const path = request.path.replace(fetchPackageFileBasePath, "");
135+
logger.info(`Fetch file for path: ${path}`);
136+
137+
const pathPackageInfo = parsePackageInfoFromPath(path);
138+
if (!pathPackageInfo) {
139+
return response.status(400).send(`Invalid package path: ${path}`);
140+
}
141+
142+
const config = NpmRegistryService.getInstance().getRegistryEntryForPackage(pathPackageInfo.packageId);
143+
fetchPackageFileInner(request, response, config);
144+
}
145+
146+
async function fetchPackageFileInner(request: ServerRequest, response: ServerResponse, config: NpmRegistryConfigEntry) {
63147
try {
64-
const path = request.path.replace(fetchPackageFileBasePath, "");
65-
logger.info(`Fetch file for path: ${path}`);
66-
148+
const path = request.path.replace(fetchPackageFileBasePath, "");
67149
const pathPackageInfo = parsePackageInfoFromPath(path);
68150
if (!pathPackageInfo) {
69151
return response.status(400).send(`Invalid package path: ${path}`);
70152
}
71153

72-
logger.info(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`);
73-
const {organization, name, version, file} = pathPackageInfo;
74-
const packageName = organization ? `@${organization}/${name}` : name;
154+
logger.debug(`Fetch file for package: ${JSON.stringify(pathPackageInfo)}`);
155+
const {packageId, version, file} = pathPackageInfo;
75156
let packageVersion = version;
76157

77158
let packageInfo: PackagesVersionInfo | null = null;
78159
if (version === "latest") {
79-
const packageInfo: PackagesVersionInfo = await fetchPackageInfo(packageName);
160+
const packageInfo: PackagesVersionInfo|null = await fetchPackageInfo(packageId, config);
161+
if (packageInfo === null) {
162+
return response.status(404).send("Not found");
163+
}
80164
packageVersion = packageInfo["dist-tags"].latest;
81165
}
166+
167+
// Wait for package to be processed if it's already being processed
168+
if (PackageProcessingQueue.has(packageId)) {
169+
logger.info("Waiting for package to be processed", packageId);
170+
await PackageProcessingQueue.wait(packageId);
171+
}
82172

83-
const packageBaseDir = `${CACHE_DIR}/${packageName}/${packageVersion}/package`;
173+
const packageBaseDir = `${CACHE_DIR}/${packageId}/${packageVersion}/package`;
84174
const packageExists = await fileExists(`${packageBaseDir}/package.json`)
85175
if (!packageExists) {
86-
if (!packageInfo) {
87-
packageInfo = await fetchPackageInfo(packageName);
88-
}
89-
90-
if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) {
91-
return response.status(404).send("Not found");
176+
try {
177+
logger.info(`Package does not exist, fetch from registy: ${packageId}@${packageVersion}`);
178+
PackageProcessingQueue.add(packageId);
179+
if (!packageInfo) {
180+
packageInfo = await fetchPackageInfo(packageId, config);
181+
}
182+
183+
if (!packageInfo || !packageInfo.versions || !packageInfo.versions[packageVersion]) {
184+
return response.status(404).send("Not found");
185+
}
186+
187+
const tarball = packageInfo.versions[packageVersion].dist.tarball;
188+
logger.info(`Fetching tarball: ${tarball}`);
189+
await fetchAndUnpackTarball(tarball, packageId, packageVersion, config);
190+
} catch (error) {
191+
logger.error("Error fetching package tarball", error);
192+
return response.status(500).send("Internal server error");
193+
} finally {
194+
PackageProcessingQueue.resolve(packageId);
92195
}
93-
94-
const tarball = packageInfo.versions[packageVersion].dist.tarball;
95-
logger.info("Fetching tarball...", tarball);
96-
await fetchAndUnpackTarball(tarball, packageName, packageVersion);
196+
} else {
197+
logger.info(`Package already exists, serve from cache: ${packageBaseDir}/${file}`)
97198
}
98199

99200
// Fallback to index.mjs if index.js is not present
100201
if (file === "index.js" && !await fileExists(`${packageBaseDir}/${file}`)) {
101-
logger.info("Fallback to index.mjs");
202+
logger.debug("Fallback to index.mjs");
102203
return response.sendFile(`${packageBaseDir}/index.mjs`);
103204
}
104205

105206
return response.sendFile(`${packageBaseDir}/${file}`);
106207
} catch (error) {
107-
logger.error("Error fetching package file",error);
208+
logger.error(`Error fetching package file: ${error}${(erroras{stack: string})?.stack?.toString()}`);
108209
response.status(500).send("Internal server error");
109210
}
110211
};
@@ -114,26 +215,22 @@ export async function fetchPackageFile(request: ServerRequest, response: ServerR
114215
* Helpers
115216
*/
116217

117-
function parsePackageInfoFromPath(path: string): {organization: string, name: string, version: string, file: string} | undefined {
118-
logger.info(`Parse package info from path: ${path}`);
218+
function parsePackageInfoFromPath(path: string): {packageId: string, organization: string, name: string, version: string, file: string} | undefined {
119219
//@ts-ignore - regex groups
120-
const packageInfoRegex = /^\/?(?<fullName>(?:@(?<organization>[a-z0-9-~][a-z0-9-._~]*)\/)?(?<name>[a-z0-9-~][a-z0-9-._~]*))(?:@(?<version>[-a-z0-9><=_.^~]+))?\/(?<file>[^\r\n]*)?$/;
220+
const packageInfoRegex = /^\/?(?<packageId>(?:@(?<organization>[a-z0-9-~][a-z0-9-._~]*)\/)?(?<name>[a-z0-9-~][a-z0-9-._~]*))(?:@(?<version>[-a-z0-9><=_.^~]+))?\/(?<file>[^\r\n]*)?$/;
121221
const matches = path.match(packageInfoRegex);
122-
logger.info(`Parse package matches: ${JSON.stringify(matches)}`);
123222
if (!matches?.groups) {
124223
return;
125224
}
126225

127-
let {organization, name, version, file} = matches.groups;
226+
let {packageId,organization, name, version, file} = matches.groups;
128227
version = /^\d+\.\d+\.\d+(-[\w\d]+)?/.test(version) ? version : "latest";
129228

130-
return {organization, name, version, file};
229+
return {packageId,organization, name, version, file};
131230
}
132231

133-
function fetchFromRegistry(packageName: string, urlOrPath: string): Promise<Response> {
134-
const config: NpmRegistryConfigEntry = NpmRegistryService.getInstance().getRegistryEntryForPackage(packageName);
232+
function fetchFromRegistry(urlOrPath: string, config: NpmRegistryConfigEntry): Promise<Response> {
135233
const registryUrl = config?.registry.url;
136-
137234
const headers: {[key: string]: string} = {};
138235
switch (config?.registry.auth.type) {
139236
case "none":
@@ -154,31 +251,35 @@ function fetchFromRegistry(packageName: string, urlOrPath: string): Promise<Resp
154251
url = `${registryUrl}${separator}${urlOrPath}`;
155252
}
156253

157-
logger.debug(`Fetch from registry: ${url}`);
254+
logger.debug(`Fetch from registry: ${url}, ${JSON.stringify(headers)}`);
158255
return fetch(url, {headers});
159256
}
160257

161-
function fetchPackageInfo(packageName: string): Promise<PackagesVersionInfo> {
162-
return fetchFromRegistry(packageName, packageName).then(res => res.json());
258+
function fetchPackageInfo(packageName: string, config: NpmRegistryConfigEntry): Promise<PackagesVersionInfo|null> {
259+
return fetchFromRegistry(`/${packageName}`, config).then(res => {
260+
if (!res.ok) {
261+
logger.error(`Failed to fetch package info for package ${packageName}: ${res.statusText}`);
262+
return null;
263+
}
264+
return res.json();
265+
});
163266
}
164267

165-
async function fetchAndUnpackTarball(url: string, packageName: string, packageVersion: string) {
166-
const response: Response = await fetchFromRegistry(packageName,url);
268+
async function fetchAndUnpackTarball(url: string, packageId: string, packageVersion: string,config: NpmRegistryConfigEntry) {
269+
const response: Response = await fetchFromRegistry(url,config);
167270
const arrayBuffer = await response.arrayBuffer();
168271
const buffer = Buffer.from(arrayBuffer);
169272
const path = `${CACHE_DIR}/${url.split("/").pop()}`;
170273
await fs.writeFile(path, buffer);
171-
await unpackTarball(path, packageName, packageVersion);
274+
await unpackTarball(path, packageId, packageVersion);
172275
await fs.unlink(path);
173276
}
174277

175-
async function unpackTarball(path: string, packageName: string, packageVersion: string) {
176-
const destinationPath = `${CACHE_DIR}/${packageName}/${packageVersion}`;
278+
async function unpackTarball(path: string, packageId: string, packageVersion: string) {
279+
const destinationPath = `${CACHE_DIR}/${packageId}/${packageVersion}`;
177280
await fs.mkdir(destinationPath, { recursive: true });
178281
await new Promise<void> ((resolve, reject) => {
179282
const tar = spawn("tar", ["-xvf", path, "-C", destinationPath]);
180-
tar.stdout.on("data", (data) => logger.info(data));
181-
tar.stderr.on("data", (data) => console.error(data));
182283
tar.on("close", (code) => {
183284
code === 0 ? resolve() : reject();
184285
});

‎server/node-service/src/routes/apiRouter.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,7 @@ apiRouter.post("/validatePluginDataSourceConfig", pluginControllers.validatePlug
1616
apiRouter.get("/npm/registry/*", npmControllers.fetchRegistry);
1717
apiRouter.get("/npm/package/*", npmControllers.fetchPackageFile);
1818

19+
apiRouter.post("/npm/registry/*", npmControllers.fetchRegistryWithConfig);
20+
apiRouter.post("/npm/package/*", npmControllers.fetchPackageFileWithConfig);
21+
1922
export default apiRouter;

‎server/node-service/src/services/npmRegistry.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,4 +108,26 @@ export class NpmRegistryService {
108108

109109
return config;
110110
}
111+
112+
public static getRegistryEntryForPackageWithConfig(packageName: string, registryConfig: NpmRegistryConfig): NpmRegistryConfigEntry {
113+
registryConfig = NpmRegistryService.sortRegistryConfig(registryConfig);
114+
const config: NpmRegistryConfigEntry | undefined = registryConfig.find(entry => {
115+
if (entry.scope.type === "organization") {
116+
return packageName.startsWith(entry.scope.pattern);
117+
} else if (entry.scope.type === "package") {
118+
return packageName === entry.scope.pattern;
119+
} else {
120+
return true;
121+
}
122+
});
123+
124+
if (!config) {
125+
logger.info(`No registry entry found for package: ${packageName}`);
126+
return NpmRegistryService.DEFAULT_REGISTRY;
127+
} else {
128+
logger.info(`Found registry entry for package: ${packageName} -> ${config.registry.url}`);
129+
}
130+
131+
return config;
132+
}
111133
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /