Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Feature/parallel fetch data #267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tstelzer wants to merge 13 commits into contentlayerdev:main
base: main
Choose a base branch
Loading
from tstelzer:feature/parallel-fetch-data
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
494ffc1
feat(@contentlayer): begin threaded building of source files
tstelzer Jul 7, 2022
71f286c
docs: test parallelism changes with gatsby docs
tstelzer Jul 7, 2022
aa764e9
docs: add TODO
tstelzer Jul 7, 2022
a0bff26
feat(@contentlayer): serialize DocumentTypeMapState
tstelzer Jul 7, 2022
266d267
build: add eslint prettier plugin
tstelzer Jul 8, 2022
d814ae1
feat(@contentlayer): make documentTypeMap explicit in makeCacheItemFr...
tstelzer Jul 8, 2022
a53243e
feat: compute makeCacheItemFromFilePath via worker pool
tstelzer Jul 8, 2022
6d8d3a2
feat: de-serializing errors
tstelzer Jul 8, 2022
d8784a6
fix: node worker (supposedly) OOM
tstelzer Jul 8, 2022
b4dcb70
style: improve
tstelzer Jul 8, 2022
fcb0103
feat: replace dummy with live tracing in worker
tstelzer Jul 8, 2022
fbd13b3
build: unserialized plugin throwing
tstelzer Jul 8, 2022
5c827a3
docs: update
tstelzer Jul 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .editorconfig
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
root = true

[*]
end_of_line = lf
insert_final_newline = true

[*.{js,jsx,ts,tsx,json,mjs}]
charset = utf-8
indent_style = space
trim_trailing_whitespace = true
indent_size = 2
5 changes: 3 additions & 2 deletions .eslintrc
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
"node": true,
"es6": true
},
"ignorePatterns": ["packages/_archive/*", "examples/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"],
"ignorePatterns": ["packages/_archive/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"],
"parser": "@typescript-eslint/parser",
"plugins": ["@typescript-eslint", "simple-import-sort", "import"],
"plugins": ["prettier", "@typescript-eslint", "simple-import-sort", "import"],
"extends": ["plugin:react-hooks/recommended", "plugin:@typescript-eslint/recommended", "prettier"],
"rules": {
"prettier/prettier": "error",
"simple-import-sort/imports": "error",
"import/no-duplicates": "warn",
// "func-style": ["warn", "expression"],
Expand Down
4 changes: 4 additions & 0 deletions examples/gatsbydocs/.gitignore
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
content
.next
.contentlayer
node_modules
80 changes: 80 additions & 0 deletions examples/gatsbydocs/contentlayer.config.ts
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import rehypeShiki from '@stefanprobst/rehype-shiki'
import type { FieldDef } from 'contentlayer/source-files'
import { defineDocumentType, defineNestedType, makeSource } from 'contentlayer/source-files'
import { createRequire } from 'module'
import * as path from 'path'
import * as shiki from 'shiki'

const Example = defineNestedType(() => ({
name: 'Example',
fields: {
label: { type: 'string' },
href: { type: 'string' },
},
}))

const fields: Record<string, FieldDef> = {
title: {
type: 'string',
required: true,
},
jsdoc: {
type: 'list',
of: { type: 'string' },
},
tableOfContentsDepth: { type: 'number' },
showTopLevelSignatures: { type: 'boolean' },
date: { type: 'string' },
version: { type: 'string' },
canonicalLink: { type: 'string' },
apiCalls: { type: 'string' },
contentsHeading: { type: 'string' },
description: { type: 'string' },
examples: {
type: 'list',
of: Example,
},
}

const Reference = defineDocumentType(() => ({
name: 'Reference',
filePathPattern: 'docs/reference/**/*.md',
fields,
}))

const HowTo = defineDocumentType(() => ({
name: 'HowTo',
filePathPattern: 'docs/how-to/**/*.md',
fields,
}))

const Conceptual = defineDocumentType(() => ({
name: 'Conceptual',
filePathPattern: 'docs/conceptual/**/*.md',
fields,
}))

const Tutorial = defineDocumentType(() => ({
name: 'Tutorial',
filePathPattern: 'tutorial/**/*.md',
fields,
}))

export default makeSource(async () => {
// const require = createRequire(import.meta.url)
// const shikiPkgPath = (dir: string) => path.join(require.resolve('shiki'), '..', '..', dir, path.sep)
// const highlighter = await shiki.getHighlighter({
// paths: { languages: shikiPkgPath('languages'), themes: shikiPkgPath('themes') },
// theme: 'github-light',
// })

return {
contentDirPath: 'content',
documentTypes: [Reference, HowTo, Conceptual, Tutorial],
// onUnknownDocuments: 'skip-ignore',
// markdown: {
// // '@stefanprobst/rehype-shiki', {}
// rehypePlugins: [[rehypeShiki, { highlighter }]],
// },
}
})
5 changes: 5 additions & 0 deletions examples/gatsbydocs/next-env.d.ts
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />

// NOTE: This file should not be edited
// see https://nextjs.org/docs/basic-features/typescript for more information.
4 changes: 4 additions & 0 deletions examples/gatsbydocs/next.config.js
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
const { withContentlayer } = require('next-contentlayer')

// module.exports = {}
module.exports = withContentlayer({})
25 changes: 25 additions & 0 deletions examples/gatsbydocs/package.json
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"name": "examples-gatsby-docs",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start"
},
"dependencies": {
"next": "12.1.6",
"react": "18.1.0",
"react-dom": "18.1.0"
},
"devDependencies": {
"@leafac/rehype-shiki": "^1.3.1",
"@stefanprobst/rehype-shiki": "^2.0.4",
"@types/react": "18.0.9",
"@types/react-dom": "^17.0.9",
"contentlayer": "workspace:*",
"eslint-config-next": "^11.0.1",
"next-contentlayer": "workspace:*",
"shiki": "^0.9.4",
"typescript": "4.6.4"
}
}
27 changes: 27 additions & 0 deletions examples/gatsbydocs/pages/[...id].tsx
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import type { DocumentTypes } from 'contentlayer/generated'
import { allDocuments } from 'contentlayer/generated'
import Head from 'next/head'
import type { FC } from 'react'

export const getStaticPaths = () => {
const paths = allDocuments.map((_) => `/${_._raw.flattenedPath}`)

return { paths, fallback: false }
}

export const getStaticProps = (context: any) => {
const doc = allDocuments.find((_) => _._raw.flattenedPath === context.params.id.join('/'))

return { props: { doc } }
}

const Page: FC<{ doc: DocumentTypes }> = ({ doc }) => (
<>
<Head>
<title>{doc.title}</title>
</Head>
<div dangerouslySetInnerHTML={{ __html: doc.body.html }} />
</>
)

export default Page
21 changes: 21 additions & 0 deletions examples/gatsbydocs/pages/index.tsx
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { allDocuments } from 'contentlayer/generated'
import type { InferGetStaticPropsType } from 'next'
import type { FC } from 'react'

export const getStaticProps = () => {
const docs = allDocuments.map((_) => ({ path: _._raw.flattenedPath, title: _.title }))

return { props: { docs } }
}

const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ docs }) => (
<div>
{docs.map((doc) => (
<a style={{ display: 'block' }} key={doc.path} href={`/${doc.path}`}>
{doc.title}
</a>
))}
</div>
)

export default Page
41 changes: 41 additions & 0 deletions examples/gatsbydocs/tsconfig.json
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{
"compilerOptions": {
"jsx": "preserve",
"allowSyntheticDefaultImports": true,
"rootDir": ".",
"allowJs": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"esModuleInterop": true,
"resolveJsonModule": true,
"incremental": false,
"composite": false,
"noEmit": true,
"target": "es5",
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"strict": false,
"module": "esnext",
"moduleResolution": "node",
"isolatedModules": true,
"baseUrl": ".",
"paths": {
"contentlayer/generated": [
"./.contentlayer/generated"
]
}
},
"include": [
"next-env.d.ts",
"**/*.tsx",
"**/*.ts",
".contentlayer/generated"
],
"exclude": [
"node_modules",
"gatsby"
]
}
1 change: 1 addition & 0 deletions package.json
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"eslint": "^8.15.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-react-hooks": "^4.5.0",
"eslint-plugin-simple-import-sort": "^7.0.0",
"prettier": "^2.6.2",
Expand Down
1 change: 1 addition & 0 deletions packages/@contentlayer/source-files/package.json
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"fast-glob": "^3.2.11",
"gray-matter": "^4.0.3",
"micromatch": "^4.0.5",
"piscina": "^3.2.0",
"ts-pattern": "^4.0.2",
"unified": "^10.1.2",
"yaml": "^1.10.2"
Expand Down
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ export const runTest = async ({
previousCache: undefined,
contentTypeMap,
}),
// TODO: DocumentTypeMap is now explicitly returned from
// makeCacheItemFromFilePath in [1]. Verify that it is tested somewhere
// else, as we're throwing it away here.
T.map((_) => _.tuple[0]),
These.effectToEither,
),
)
Expand Down
41 changes: 41 additions & 0 deletions packages/@contentlayer/source-files/src/errors/index.ts
View file Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,47 @@ This is possibly a bug in Contentlayer. Please open an issue.`

renderLine = () => `"${this.documentFilePath}": ${errorToString(this.error)}`
}

const tagToError = {
InvalidFrontmatterError: InvalidFrontmatterError,
InvalidMarkdownFileError: InvalidMarkdownFileError,
InvalidYamlFileError: InvalidYamlFileError,
InvalidJsonFileError: InvalidJsonFileError,
ComputedValueError: ComputedValueError,
UnsupportedFileExtension: UnsupportedFileExtension,
FileExtensionMismatch: FileExtensionMismatch,
NoSuchDocumentTypeError: NoSuchDocumentTypeError,
NoSuchNestedDocumentTypeError: NoSuchNestedDocumentTypeError,
CouldNotDetermineDocumentTypeError: CouldNotDetermineDocumentTypeError,
MissingRequiredFieldsError: MissingRequiredFieldsError,
ExtraFieldDataError: ExtraFieldDataError,
ReferencedFileDoesNotExistError: ReferencedFileDoesNotExistError,
IncompatibleFieldDataError: IncompatibleFieldDataError,
SingletonDocumentNotFoundError: SingletonDocumentNotFoundError,
UnexpectedError: UnexpectedError,
}

export const fromSerialized = ({
_tag,
...rest
}: {
_tag: keyof typeof tagToError
// FIXME: Obviously, this type is a lie, arguments would be a union of
// class params, instead of an intersection.
readonly error: unknown
readonly documentFilePath: PosixFilePath
readonly documentTypeName: string
readonly fieldName: string
readonly extension: string
readonly filePath: string
readonly typeFieldName: string
readonly contentType: DocumentContentType
readonly referencedFilePath: PosixFilePath
readonly validNestedTypeNames: readonly string[]
readonly fieldDefsWithMissingData: core.FieldDef[]
readonly extraFieldEntries: readonly (readonly [fieldKey: string, fieldValue: any])[]
readonly incompatibleFieldData: readonly (readonly [fieldKey: string, fieldValue: any])[]
}): FetchDataError => new tagToError[_tag](rest)
}

export type SchemaError = DuplicateBodyFieldError
Expand Down
View file Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import type { PosixFilePath } from '@contentlayer/utils'
import type { Has } from '@contentlayer/utils/effect'
import { HashMap, O, pipe, State, T, Tagged } from '@contentlayer/utils/effect'
import type { Has, Option, These } from '@contentlayer/utils/effect'
import { HashMap, O, pipe, State, T, Tagged, Tuple } from '@contentlayer/utils/effect'

type DocumentTypeName = string
export type DocumentTypeName = string

export class DocumentTypeMap extends Tagged('@local/DocumentTypeMap')<{
readonly map: HashMap.HashMap<DocumentTypeName, PosixFilePath[]>
Expand All @@ -29,6 +29,21 @@ export class DocumentTypeMap extends Tagged('@local/DocumentTypeMap')<{
*/
export const DocumentTypeMapState = State.State<DocumentTypeMap>(DocumentTypeMap._tag)

export const serialize = () =>
pipe(
DocumentTypeMapState.get,
// FIXME: unsafe
T.chain((map) => T.succeedWith(() => JSON.stringify(map))),
)

export function provideFromSerialized(serialized: string) {
return pipe(
// FIXME: unsafe
T.succeedWith<DocumentTypeMap>(() => JSON.parse(serialized)),
T.map((map) => T.provideSomeLayer(DocumentTypeMapState.Live(new DocumentTypeMap(map)))),
)
}

export const provideDocumentTypeMapState = T.provideSomeLayer(DocumentTypeMapState.Live(DocumentTypeMap.init()))

export type HasDocumentTypeMapState = Has<State.State<DocumentTypeMap>>
Loading

AltStyle によって変換されたページ (->オリジナル) /