JIT data (#32140)

Author: Peter Bengtsson

components/context/MainContext.tsx

@@ -40,7 +40,7 @@ type DataT = {
       version_was_deprecated: string
       version_will_be_deprecated: string
       deprecation_details: string
-      isOldestReleaseDeprecated: boolean
+      isOldestReleaseDeprecated?: boolean
     }
     policies: {
       translation: string
@@ -130,12 +130,27 @@ export const getMainContext = async (req: any, res: any): Promise<MainContextT>
     error: req.context.error ? req.context.error.toString() : '',
     data: {
       ui: req.context.site.data.ui,
+
       reusables: {
-        enterprise_deprecation: req.context.site.data.reusables.enterprise_deprecation,
-        policies: req.context.site.data.reusables.policies,
+        enterprise_deprecation: {
+          version_was_deprecated: req.context.getDottedData(
+            'reusables.enterprise_deprecation.version_was_deprecated'
+          ),
+          version_will_be_deprecated: req.context.getDottedData(
+            'reusables.enterprise_deprecation.version_will_be_deprecated'
+          ),
+          deprecation_details: req.context.getDottedData(
+            'reusables.enterprise_deprecation.deprecation_details'
+          ),
+        },
+        policies: {
+          translation: req.context.getDottedData('reusables.policies.translation'),
+        },
       },
       variables: {
-        release_candidate: req.context.site.data.variables.release_candidate,
+        release_candidate: {
+          version: req.context.getDottedData('variables.release_candidate.version') || null,
+        },
       },
     },
     currentCategory: req.context.currentCategory || '',

content/get-started/quickstart/github-glossary.md

@@ -13,7 +13,7 @@ versions:
 ---
 {% for glossary in glossaries %}
   ### {{ glossary.term }}
-  {{ glossary.description}}
+  {{ glossary.description }}
   ---
 {% endfor %}
 

lib/get-applicable-versions.js

@@ -1,12 +1,8 @@
-import path from 'path'
 import { reduce, sortBy } from 'lodash-es'
 import { allVersions } from './all-versions.js'
 import versionSatisfiesRange from './version-satisfies-range.js'
 import checkIfNextVersionOnly from './check-if-next-version-only.js'
-import dataDirectory from './data-directory.js'
-import encodeBracketedParentheses from './encode-bracketed-parentheses.js'
-
-const featuresDir = path.join('data', 'features')
+import { getDeepDataByLanguage } from './get-data.js'
 
 let featureData = null
 
@@ -24,10 +20,7 @@ function getApplicableVersions(frontmatterVersions, filepath) {
   }
 
   if (!featureData) {
-    featureData = dataDirectory(featuresDir, {
-      preprocess: (dataString) => encodeBracketedParentheses(dataString.trimEnd()),
-      ignorePatterns: [/README\.md$/],
-    })
+    featureData = getDeepDataByLanguage('features', 'en')
   }
 
   // Check for frontmatter that includes a feature name, like:

lib/get-data.js

@@ -7,6 +7,10 @@ import { merge, get } from 'lodash-es'
 
 import languages from './languages.js'
 
+// If you run `export DEBUG_JIT_DATA_READS=true` in your terminal,
+// next time it will mention every file it reads from disk.
+const DEBUG_JIT_DATA_READS = Boolean(JSON.parse(process.env.DEBUG_JIT_DATA_READS || 'false'))
+
 // This is a list of files that we should always immediately fall back to
 // English for.
 // Having this is safer than trying to wrangle the translations to NOT
@@ -237,7 +241,7 @@ const getMarkdownContent = memoize((root, relPath, englishRoot) => {
 
 const getFileContent = (root, relPath, englishRoot) => {
   const filePath = root ? path.join(root, relPath) : relPath
-  if (process.env.NODE_ENV === 'development') console.log('READ', filePath)
+  if (DEBUG_JIT_DATA_READS) console.log('READ', filePath)
   try {
     return fs.readFileSync(filePath, 'utf-8')
   } catch (err) {

lib/get-english-headings.js

@@ -1,7 +1,9 @@
 import { fromMarkdown } from 'mdast-util-from-markdown'
 import { toString } from 'mdast-util-to-string'
 import { visit } from 'unist-util-visit'
+
 import findPage from './find-page.js'
+import { getDataByLanguage } from './get-data.js'
 
 // for any translated page, first get corresponding English markdown
 // then get the headings on both the translated and English pageMap
@@ -11,7 +13,7 @@ export default function getEnglishHeadings(page, context) {
   // generated programatically.
   if (page.relativePath.endsWith('/github-glossary.md')) {
     // Return an object of `{ localized-term: english-slug }`
-    const languageGlossary = context.site.data.glossaries.external
+    const languageGlossary = getDataByLanguage('glossaries.external', 'en')
     return languageGlossary.reduce((prev, curr) => {
       prev[curr.term] = curr.slug
       return prev

lib/liquid-tags/data.js

@@ -1,7 +1,7 @@
 import { TokenizationError } from 'liquidjs'
-import matter from 'gray-matter'
 
 import { THROW_ON_EMPTY, DataReferenceError } from './error-handling.js'
+import { getDataByLanguage } from '../get-data.js'
 
 const Syntax = /([a-z0-9/\\_.\-[\]]+)/i
 const SyntaxHelp = "Syntax Error in 'data' - Valid syntax: data [path]"
@@ -16,8 +16,8 @@ export default {
   },
 
   async render(scope) {
-    let value = await this.liquid.evalValue(`site.data.${this.path}`, scope)
-    if (typeof value !== 'string') {
+    const text = getDataByLanguage(this.path, scope.environments.currentLanguage)
+    if (text === undefined) {
       const message = `Can't find the key 'site.data.${this.path}' in the scope.`
       if (THROW_ON_EMPTY) {
         throw new DataReferenceError(message)
@@ -26,12 +26,6 @@ export default {
       return
     }
 
-    // Drop any frontmatter since reusable Markdown files aren't currently
-    // expected to use frontmatter.  This is useful since our current translation
-    // process adds YAML frontmatter properties to any Markdown file that gets
-    // translated and we don't want to render that information.
-    ;({ content: value } = matter(value))
-
-    return this.liquid.parseAndRender(value, scope.environments)
+    return this.liquid.parseAndRender(text, scope.environments)
   },
 }

lib/liquid-tags/indented-data-reference.js

@@ -1,6 +1,7 @@
 import assert from 'assert'
 
 import { THROW_ON_EMPTY, IndentedDataReferenceError } from './error-handling.js'
+import { getDataByLanguage } from '../get-data.js'
 
 // This class supports a tag that expects two parameters, a data reference and `spaces=NUMBER`:
 //
@@ -33,20 +34,8 @@ export default {
     assert(parseInt(numSpaces) || numSpaces === '0', '"spaces=NUMBER" must include a number')
 
     // Get the referenced value from the context
-    const value = await this.liquid.evalValue(`site.data.${dataReference}`, scope)
-
-    // If value is falsy it can be because we completely failed to look
-    // it up. But it can also be literally an empty string.
-    // For example, the reusable could be entirely something like this:
-    //
-    //   {% if some condition %}The meat{% endif %}
-    //
-    // Then it's working as expected. But if the reference is wrong, e.g.
-    //
-    //   {% indented_data_reference reusables.foo.tyypu spaces=3 %}
-    //
-    // Or if the file simple doesn't exist, you get an undefined for the value.
-    if (typeof value !== 'string' && !value) {
+    const text = getDataByLanguage(dataReference, scope.environments.currentLanguage)
+    if (text === undefined) {
       const message = `Can't find the key 'site.data.${dataReference}' in the scope.`
       if (THROW_ON_EMPTY) {
         throw new IndentedDataReferenceError(message)
@@ -56,7 +45,7 @@ export default {
     }
 
     // add spaces to each line
-    const renderedReferenceWithIndent = value.replace(/^/gm, ' '.repeat(numSpaces))
+    const renderedReferenceWithIndent = text.replace(/^/gm, ' '.repeat(numSpaces))
 
     return this.liquid.parseAndRender(renderedReferenceWithIndent, scope.environments)
   },

lib/page-data.js

@@ -1,8 +1,8 @@
 import path from 'path'
+
 import languages from './languages.js'
 import { allVersions } from './all-versions.js'
 import createTree, { getBasePath } from './create-tree.js'
-import loadSiteData from './site-data.js'
 import nonEnterpriseDefaultVersion from './non-enterprise-default-version.js'
 import Page from './page.js'
 
@@ -60,8 +60,7 @@ export async function loadUnversionedTree(languagesOnly = null) {
  *
  * Order of languages and versions doesn't matter, but order of child page arrays DOES matter (for navigation).
 */
-export async function loadSiteTree(unversionedTree, siteData) {
-  const site = siteData || loadSiteData()
+export async function loadSiteTree(unversionedTree) {
   const rawTree = Object.assign({}, unversionedTree || (await loadUnversionedTree()))
   const siteTree = {}
 
@@ -76,8 +75,7 @@ export async function loadSiteTree(unversionedTree, siteData) {
           treePerVersion[version] = await versionPages(
             Object.assign({}, rawTree[langCode]),
             version,
-            langCode,
-            site
+            langCode
           )
         })
       )
@@ -89,7 +87,7 @@ export async function loadSiteTree(unversionedTree, siteData) {
   return siteTree
 }
 
-export async function versionPages(obj, version, langCode, site) {
+export async function versionPages(obj, version, langCode) {
   // Add a versioned href as a convenience for use in layouts.
   obj.href = obj.page.permalinks.find(
     (pl) =>
@@ -103,7 +101,7 @@ export async function versionPages(obj, version, langCode, site) {
       // Drop child pages that do not apply to the current version
       .filter((childPage) => childPage.page.applicableVersions.includes(version))
       // Version the child pages recursively.
-      .map((childPage) => versionPages(Object.assign({}, childPage), version, langCode, site))
+      .map((childPage) => versionPages(Object.assign({}, childPage), version, langCode))
   )
 
   obj.childPages = [...versionedChildPages]

lib/process-learning-tracks.js

@@ -1,6 +1,7 @@
 import renderContent from './render-content/index.js'
 import getLinkData from './get-link-data.js'
 import getApplicableVersions from './get-applicable-versions.js'
+import { getDataByLanguage } from './get-data.js'
 
 const renderOpts = { textOnly: true, encodeEntities: true }
 
@@ -23,11 +24,20 @@ export default async function processLearningTracks(rawLearningTracks, context)
     if (!renderedTrackName) continue
 
     // Find the data for the current product and track name.
-    const trackDataForProduct = context.site.data['learning-tracks'][context.currentProduct]
-    if (!trackDataForProduct) {
-      throw new Error(`No learning track data for product "${context.currentProduct}".`)
+
+    if (context.currentProduct.includes('.')) {
+      throw new Error(`currentProduct can not contain a . (${context.currentProduct})`)
+    }
+    if (renderedTrackName.includes('.')) {
+      throw new Error(`renderedTrackName can not contain a . (${renderedTrackName})`)
     }
-    const track = trackDataForProduct[renderedTrackName]
+
+    // Note: this will use the translated learning tracks and automatically
+    // fall back to English if they don't exist on disk in the translation.
+    const track = getDataByLanguage(
+      `learning-tracks.${context.currentProduct}.${renderedTrackName}`,
+      context.currentLanguage
+    )
     if (!track) {
       throw new Error(`No learning track called '${renderedTrackName}'.`)
     }

lib/warm-server.js

@@ -1,7 +1,6 @@
 import statsd from './statsd.js'
 import { loadUnversionedTree, loadSiteTree, loadPages, loadPageMap } from './page-data.js'
 import loadRedirects from './redirects/precompile.js'
-import loadSiteData from './site-data.js'
 
 // Instrument these functions so that
 // it's wrapped in a timer that reports to Datadog
@@ -11,7 +10,6 @@ const dog = {
   loadPages: statsd.asyncTimer(loadPages, 'load_pages'),
   loadPageMap: statsd.asyncTimer(loadPageMap, 'load_page_map'),
   loadRedirects: statsd.asyncTimer(loadRedirects, 'load_redirects'),
-  loadSiteData: statsd.timer(loadSiteData, 'load_site_data'),
 }
 
 // For multiple-triggered Promise sharing
@@ -25,8 +23,7 @@ async function warmServer() {
   }
 
   const unversionedTree = await dog.loadUnversionedTree()
-  const site = dog.loadSiteData()
-  const siteTree = await dog.loadSiteTree(unversionedTree, site)
+  const siteTree = await dog.loadSiteTree(unversionedTree)
   const pageList = await dog.loadPages(unversionedTree)
   const pageMap = await dog.loadPageMap(pageList)
   const redirects = await dog.loadRedirects(pageList)
@@ -43,7 +40,6 @@ async function warmServer() {
 
   return {
     pages: pageMap,
-    site,
     redirects,
     unversionedTree,
     siteTree,