Automatically generate pages on the wiki for the community extensions

newIDE/app/scripts/extract-extensions-document.js

@@ -6,22 +6,69 @@
 const fs = require('fs').promises;
 const { default: axios } = require('axios');
 const path = require('path');
+const {
+  gdevelopWikiUrlRoot,
+  getHelpLink,
+  generateReadMoreLink,
+  getExtensionFolderName,
+  improperlyFormattedHelpPaths,
+} = require('./lib/WikiHelpLink');
+const { convertMarkdownToDokuWikiMarkdown } = require('./lib/DokuwikiHelpers');
 
-const extensionsUrl =
-  'https://api.gdevelop-app.com/asset/extension';
+const extensionsUrl = 'https://api.gdevelop-app.com/asset/extension';
 const gdRootPath = path.join(__dirname, '..', '..', '..');
 const outputRootPath = path.join(gdRootPath, 'docs-wiki');
-const outputFilePath = path.join(outputRootPath, 'extensions.txt');
+const extensionsRootPath = path.join(outputRootPath, 'extensions');
+const extensionsFilePath = path.join(outputRootPath, 'extensions.txt');
+
+const generateSvgImageIcon = iconUrl => {
+  // Use the `&.png?` syntax to force Dokuwiki to display the image.
+  // See https://www.dokuwiki.org/images.
+  return `{{${iconUrl}?&.png?nolink&48x48 |}}`;
+};
+
+/** @returns {string} */
+const generateExtensionFooterText = (fullName) => {
+  return `
+---
+*This page is an auto-generated reference page about the **${fullName}** extension, made by the community of [[https://gdevelop-app.com/|GDevelop, the open-source, cross-platform game engine designed for everyone]].*` +
+      ' ' +
+      'Learn more about [[gdevelop5:extensions|all GDevelop community-made extensions here]].';
+};
+
+const getAllExtensions = async () => {
+  const response = await axios.get(extensionsUrl);
+  if (!response.data.databaseUrl) {
+    throw new Error('Unexpected response from the extension endpoint.');
+  }
+  const databaseResponse = await axios.get(response.data.databaseUrl);
+  const extensionsDatabase = databaseResponse.data;
+
+  const extensions = await Promise.all(
+    extensionsDatabase.extensionShortHeaders.map(async extensionShortHeader => {
+      const response = await axios.get(extensionShortHeader.url);
+      const extensionHeader = response.data;
+      if (!extensionHeader) {
+        throw new Error(
+          `Unexpected response when fetching an extension (${
+            extensionShortHeader.url
+          }).`
+        );
+      }
+
+      return extensionHeader;
+    })
+  );
+
+  return extensions;
+};
 
 (async () => {
   try {
-    const response = await axios.get(extensionsUrl);
-    if (!response.data.databaseUrl) {
-      throw new Error('Unexpected response from the extension endpoint.');
-    }
+    console.info(`ℹ️ Loading all community extensions...`);
+    const extensions = await getAllExtensions();
 
-    const databaseResponse = await axios.get(response.data.databaseUrl);
-    let texts = `# Extensions
+    let indexPageContent = `# Extensions
 
 GDevelop is built in a flexible way. In addition to [[gdevelop5:all-features|core features]], new capabilities are provided by extensions. Extensions can contain objects, behaviors, actions, conditions, expressions or events.
 
@@ -29,21 +76,55 @@ GDevelop is built in a flexible way. In addition to [[gdevelop5:all-features|cor
 
 `;
 
-    databaseResponse.data.extensionShortHeaders.forEach(element => {
-      // TODO: link to help
-      texts +=
+    for (extension of extensions) {
+      const folderName = getExtensionFolderName(extension.name);
+      const referencePageUrl = `${gdevelopWikiUrlRoot}/extensions/${folderName}/reference`;
+      const helpPageUrl = getHelpLink(extension.helpPath) || referencePageUrl;
+
+      const referencePageContent =
+        `# ${extension.fullName}` +
+        '\n\n' +
+        generateSvgImageIcon(extension.previewIconUrl) +
+        '\n' +
+        extension.shortDescription +
+        '\n\n' +
+        `**Authors and contributors** to this community extension: ${extension.author}.\n` +
+        '---\n' +
+        '\n' +
+        convertMarkdownToDokuWikiMarkdown(extension.description) +
+        '\n' +
+        generateExtensionFooterText(extension.fullName);
+
+      const extensionReferenceFilePath = path.join(
+        extensionsRootPath,
+        folderName,
+        'reference.txt'
+      );
+      await fs.mkdir(path.dirname(extensionReferenceFilePath), {
+        recursive: true,
+      });
+      await fs.writeFile(extensionReferenceFilePath, referencePageContent);
+      console.info(`ℹ️ File generated: ${extensionReferenceFilePath}`);
+
+      indexPageContent +=
         '## ' +
-        element.fullName +
+        extension.fullName +
         '\n' +
         // Use the `&.png?` syntax to force Dokuwiki to display the image.
         // See https://www.dokuwiki.org/images.
-        `{{${element.previewIconUrl}?&.png?nolink&48x48 |}}` +
+        generateSvgImageIcon(extension.previewIconUrl) +
         '\n' +
-        element.shortDescription +
+        extension.shortDescription +
+        '\n\n' +
+        // Link to help page or to reference if none.
+        `[[${helpPageUrl}|Read more...]]` +
+        (helpPageUrl !== referencePageUrl
+          ? ` ([[${referencePageUrl}|reference]])`
+          : '') +
         '\n\n';
-    });
+    }
 
-    texts += `
+    indexPageContent += `
 ## Make your own extension
 
 It's easy to create, directly in your project, new behaviors, actions, conditions or expressions for your game.
@@ -52,15 +133,24 @@ Read more about this:
 
 * [[gdevelop5:extensions:create|Create your own extensions]]
 * [[gdevelop5:extensions:share|Share extensions with the community]]
-* [[gdevelop5:extensions:extend-gdevelop|Extend GDevelop with JavaScript or C++]]`
+* [[gdevelop5:extensions:extend-gdevelop|Extend GDevelop with JavaScript or C++]]`;
 
     try {
-      await fs.mkdir(path.dirname(outputFilePath), { recursive: true });
-      await fs.writeFile(outputFilePath, texts);
-      console.info(`✅ Done. File generated: ${outputFilePath}`);
+      await fs.mkdir(path.dirname(extensionsFilePath), { recursive: true });
+      await fs.writeFile(extensionsFilePath, indexPageContent);
+      console.info(`✅ Done. File generated: ${extensionsFilePath}`);
     } catch (err) {
       console.error('❌ Error while writing output', err);
     }
+
+    if (improperlyFormattedHelpPaths.size > 0) {
+      console.info(
+        `⚠️ Extensions documents generated, but some help paths are invalid:`,
+        improperlyFormattedHelpPaths.keys()
+      );
+    } else {
+      console.info(`✅ Extensions documents generated.`);
+    }
   } catch (err) {
     console.error('❌ Error while fetching data', err);
   }

newIDE/app/scripts/extract-reference-document.js

@@ -8,18 +8,17 @@ const makeExtensionsLoader = require('./lib/LocalJsExtensionsLoader');
 const fs = require('fs').promises;
 const path = require('path');
 const shell = require('shelljs');
+const {
+  gdevelopWikiUrlRoot,
+  getHelpLink,
+  generateReadMoreLink,
+  improperlyFormattedHelpPaths,
+  getExtensionFolderName,
+} = require('./lib/WikiHelpLink');
 
 shell.exec('node import-GDJS-Runtime.js');
 
 const ignoredExtensionNames = ['BuiltinJoystick'];
-const renamedExtensionNames = {
-  AdMob: 'Admob',
-  BuiltinFile: 'Storage',
-  FileSystem: 'Filesystem',
-  TileMap: 'Tilemap',
-  BuiltinMouse: 'MouseTouch',
-};
-const gdevelopWikiUrlRoot = 'http://wiki.compilgames.net/doku.php/gdevelop5';
 const gdRootPath = path.join(__dirname, '..', '..', '..');
 const outputRootPath = path.join(gdRootPath, 'docs-wiki');
 const allFeaturesRootPath = path.join(outputRootPath, 'all-features');
@@ -28,7 +27,6 @@ const expressionsFilePath = path.join(
   allFeaturesRootPath,
   'expressions-reference.txt'
 );
-const improperlyFormattedHelpPaths = new Set();
 
 // Types definitions used in this script:
 
@@ -69,14 +67,6 @@ const improperlyFormattedHelpPaths = new Set();
  * @prop {Array<BehaviorReference>} behaviorReferences Reference of all extension behaviors.
  */
 
-/** @param {string} str */
-const toKebabCase = str => {
-  return str
-    .replace(/([a-z])([A-Z])/g, '$1-$2') // get all lowercase letters that are near to uppercase ones
-    .replace(/[\s_]+/g, '-') // replace all spaces and low dash
-    .toLowerCase(); // convert to lower case
-};
-
 const sanitizeExpressionDescription = str => {
   return (
     str
@@ -146,47 +136,6 @@ const generateExtensionSeparatorText = () => {
   };
 };
 
-/**
- * @param {string} path
- * @returns {string}
- */
-const getHelpLink = path => {
-  if (!path) return '';
-
-  /**
-   * @param {string} path
-   */
-  const isRelativePathToDocumentationRoot = path => {
-    return path.startsWith('/');
-  };
-
-  /**
-   * @param {string} path
-   */
-  const isDocumentationAbsoluteUrl = path => {
-    return path.startsWith('http://') || path.startsWith('https://');
-  };
-
-  if (isRelativePathToDocumentationRoot(path))
-    return `http://wiki.compilgames.net/doku.php/gdevelop5${path}`;
-
-  if (isDocumentationAbsoluteUrl(path)) return path;
-
-  improperlyFormattedHelpPaths.add(path);
-  return '';
-};
-
-/**
- * @param {string} helpPagePath
- * @returns {string}
- */
-const generateReadMoreLink = helpPagePath => {
-  const url = getHelpLink(helpPagePath);
-  if (!url) return '';
-
-  return `[[${url}|Read more explanations about it.]]`;
-};
-
 /** @returns {RawText} */
 const generateObjectHeaderText = ({
   extension,
@@ -286,17 +235,6 @@ const generateHeader = ({ headerName, depth }) => {
   };
 };
 
-/**
- * @param {any} extension
- * @returns {string}
- */
-const getExtensionFolderName = extension => {
-  const originalName = extension.getName();
-  return toKebabCase(
-    renamedExtensionNames[originalName] || originalName.replace(/^Builtin/, '')
-  );
-};
-
 /** @returns {ReferenceText} */
 const generateInstructionReferenceRowsText = ({
   instructionType,
@@ -625,7 +563,7 @@ Remember that you can also [[gdevelop5:extensions|search for new features in the
       })
       .flatMap(extensionReferences => {
         const folderName = getExtensionFolderName(
-          extensionReferences.extension
+          extensionReferences.extension.getName()
         );
         const helpPagePath = extensionReferences.extension.getHelpPath();
         const referencePageUrl = `${gdevelopWikiUrlRoot}/all-features/${folderName}/reference`;
@@ -675,7 +613,9 @@ const generateExtensionRawTexts = extensionReferences => {
       };
 
       return {
-        folderName: getExtensionFolderName(extensionReference.extension),
+        folderName: getExtensionFolderName(
+          extensionReference.extension.getName()
+        ),
         texts: [
           generateExtensionHeaderText({ extension, depth: 1 }),
           ...withHeaderIfNotEmpty(freeActionsReferenceTexts, {

newIDE/app/scripts/lib/DokuwikiHelpers.js

@@ -0,0 +1,26 @@
+// @ts-check
+
+/**
+ * Convert the markdown text into markdown supported by DokuWiki, converting
+ * the links and image references.
+ * @param {string} markdownText
+ * @returns {string}
+ */
+const convertMarkdownToDokuWikiMarkdown = markdownText => {
+  const markdown = markdownText
+    .replace(/\!\[(.*?)\]\((.*?)\)/g, (match, linkText, linkUrl) => {
+      const url = linkUrl.replace(/^\/+/, '');
+      const title = linkText.replace(/^\[(.*?)\]/, '$1');
+      return `{{${url}?nolink |}}`;
+    })
+    .replace(/\[(.*?)\]\((.*?)\)/g, (match, linkText, linkUrl) => {
+      const url = linkUrl.replace(/^\/+/, '');
+      const title = linkText.replace(/^\[(.*?)\]/, '$1');
+      return `{{${url}|${title}}}`;
+    });
+  return markdown;
+};
+
+module.exports = {
+  convertMarkdownToDokuWikiMarkdown,
+};

newIDE/app/scripts/lib/WikiHelpLink.js

@@ -0,0 +1,76 @@
+// @ts-check
+
+const gdevelopWikiUrlRoot = 'http://wiki.compilgames.net/doku.php/gdevelop5';
+const improperlyFormattedHelpPaths = new Set();
+
+/** @param {string} str */
+const toKebabCase = str => {
+  return str
+    .replace(/([a-z])([A-Z])/g, '$1-$2') // get all lowercase letters that are near to uppercase ones
+    .replace(/[\s_]+/g, '-') // replace all spaces and low dash
+    .toLowerCase(); // convert to lower case
+};
+
+/**
+ * @param {string} path
+ * @returns {string}
+ */
+const getHelpLink = path => {
+  if (!path) return '';
+
+  /** @param {string} path */
+  const isRelativePathToDocumentationRoot = path => {
+    return path.startsWith('/');
+  };
+
+  /** @param {string} path */
+  const isDocumentationAbsoluteUrl = path => {
+    return path.startsWith('http://') || path.startsWith('https://');
+  };
+
+  if (isRelativePathToDocumentationRoot(path))
+    return `${gdevelopWikiUrlRoot}${path}`;
+
+  if (isDocumentationAbsoluteUrl(path)) return path;
+
+  improperlyFormattedHelpPaths.add(path);
+  return '';
+};
+
+/**
+ * @param {string} helpPagePath
+ * @returns {string}
+ */
+const generateReadMoreLink = helpPagePath => {
+  const url = getHelpLink(helpPagePath);
+  if (!url) return '';
+
+  return `[[${url}|Read more explanations about it.]]`;
+};
+
+const renamedExtensionNames = {
+  AdMob: 'Admob',
+  BuiltinFile: 'Storage',
+  FileSystem: 'Filesystem',
+  TileMap: 'Tilemap',
+  BuiltinMouse: 'MouseTouch',
+};
+
+/**
+ * @param {string} extensionName
+ * @returns {string}
+ */
+const getExtensionFolderName = extensionName => {
+  return toKebabCase(
+    renamedExtensionNames[extensionName] ||
+      extensionName.replace(/^Builtin/, '')
+  );
+};
+
+module.exports = {
+  gdevelopWikiUrlRoot,
+  improperlyFormattedHelpPaths,
+  getHelpLink,
+  generateReadMoreLink,
+  getExtensionFolderName,
+};

newIDE/app/src/Utils/GDevelopServices/Extension.js

@@ -19,6 +19,7 @@ export type ExtensionShortHeader = {|
 |};
 export type ExtensionHeader = {|
   ...ExtensionShortHeader,
+  helpPath: string,
   description: string,
   iconUrl: string,
 |};