Interface data storage in PROGMEM (#71)

Adds a webpack plugin to package interface as PROGMEM into a header file in the framework.
Adds a build flag to optionally enable serving from PROGMEM or SPIFFS as required
Adds documentation changes to describe changes

interface/config-overrides.js

@@ -1,7 +1,8 @@
 const ManifestPlugin = require('webpack-manifest-plugin');
 const WorkboxWebpackPlugin = require('workbox-webpack-plugin');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
-const CompressionPlugin = require("compression-webpack-plugin");
+const CompressionPlugin = require('compression-webpack-plugin');
+const ProgmemGenerator = require('./progmem-generator.js');
 
 const path = require('path');
 const fs = require('fs');
@@ -21,6 +22,9 @@ module.exports = function override(config, env) {
     miniCssExtractPlugin.options.filename = "css/[id].[contenthash:4].css";
     miniCssExtractPlugin.options.chunkFilename = "css/[id].[contenthash:4].c.css";
 
+    // build progmem data files
+    config.plugins.push(new ProgmemGenerator({ outputPath: "../lib/framework/WWWData.h", bytesPerLine: 20 }));
+
     // add compression plugin, compress javascript
     config.plugins.push(new CompressionPlugin({
       filename: "[path].gz[query]",

interface/package-lock.json

@@ -8598,11 +8598,18 @@
       "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA=="
     },
     "mime-types": {
-      "version": "2.1.24",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
-      "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
+      "version": "2.1.25",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.25.tgz",
+      "integrity": "sha512-5KhStqB5xpTAeGqKBAMgwaYMnQik7teQN4IAzC7npDv6kzeU6prfkR67bc87J1kWMPGkoaZSq1npmexMgkmEVg==",
       "requires": {
-        "mime-db": "1.40.0"
+        "mime-db": "1.42.0"
+      },
+      "dependencies": {
+        "mime-db": {
+          "version": "1.42.0",
+          "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.42.0.tgz",
+          "integrity": "sha512-UbfJCR4UAVRNgMpfImz05smAXK7+c+ZntjaA26ANtkXLlOe947Aag5zdIcKQULAiF9Cq4WxBi9jUs5zkA84bYQ=="
+        }
       }
     },
     "mimic-fn": {
@@ -13483,6 +13490,11 @@
         "camelcase": "^5.0.0",
         "decamelize": "^1.2.0"
       }
+    },
+    "zlib": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",
+      "integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA="
     }
   }
 }

interface/package.json

@@ -7,6 +7,7 @@
     "@material-ui/icons": "^4.5.1",
     "compression-webpack-plugin": "^2.0.0",
     "jwt-decode": "^2.2.0",
+    "mime-types": "^2.1.25",
     "moment": "^2.24.0",
     "notistack": "^0.9.6",
     "prop-types": "^15.7.2",
@@ -17,11 +18,12 @@
     "react-material-ui-form-validator": "^2.0.9",
     "react-router": "^5.1.1",
     "react-router-dom": "^5.1.1",
-    "react-scripts": "3.0.1"
+    "react-scripts": "3.0.1",
+    "zlib": "^1.0.5"
   },
   "scripts": {
     "start": "react-app-rewired start",
-    "build": "react-app-rewired build && rm -rf ../data/www && cp -r build ../data/www",
+    "build": "react-app-rewired build",
     "test": "react-app-rewired test --env=jsdom",
     "eject": "react-scripts eject"
   },

interface/progmem-generator.js

@@ -0,0 +1,122 @@
+const { resolve, relative, sep } = require('path');
+const { readdirSync, existsSync, unlinkSync, readFileSync, createWriteStream } = require('fs');
+var zlib = require('zlib');
+var mime = require('mime-types');
+
+const ARDUINO_INCLUDES = "#include <Arduino.h>\n\n";
+
+function getFilesSync(dir, files = []) {
+  readdirSync(dir, { withFileTypes: true }).forEach(entry => {
+    const entryPath = resolve(dir, entry.name);
+    if (entry.isDirectory()) {
+      getFilesSync(entryPath, files);
+    } else {
+      files.push(entryPath);
+    }
+  })
+  return files;
+}
+
+function coherseToBuffer(input) {
+  return Buffer.isBuffer(input) ? input : Buffer.from(input);
+}
+
+function cleanAndOpen(path) {
+  if (existsSync(path)) {
+    unlinkSync(path);
+  }
+  return createWriteStream(path, { flags: "w+" });
+}
+
+class ProgmemGenerator {
+
+  constructor(options = {}) {
+    const { outputPath, bytesPerLine = 20, indent = "  ", includes = ARDUINO_INCLUDES } = options;
+    this.options = { outputPath, bytesPerLine, indent, includes };
+  }
+
+  apply(compiler) {
+    compiler.hooks.emit.tapAsync(
+      { name: 'ProgmemGenerator' },
+      (compilation, callback) => {
+        const { outputPath, bytesPerLine, indent, includes } = this.options;
+        const fileInfo = [];
+        const writeStream = cleanAndOpen(resolve(compilation.options.context, outputPath));
+        try {
+          const writeIncludes = () => {
+            writeStream.write(includes);
+          }
+
+          const writeFile = (relativeFilePath, buffer) => {
+            const variable = "ESP_REACT_DATA_" + fileInfo.length;
+            const mimeType = mime.lookup(relativeFilePath);
+            var size = 0;
+            writeStream.write("const uint8_t " + variable + "[] PROGMEM = {");
+            const zipBuffer = zlib.gzipSync(buffer);
+            zipBuffer.forEach((b) => {
+              if (!(size % bytesPerLine)) {
+                writeStream.write("\n");
+                writeStream.write(indent);
+              }
+              writeStream.write("0x" + ("00" + b.toString(16).toUpperCase()).substr(-2) + ",");
+              size++;
+            });
+            if (size % bytesPerLine) {
+              writeStream.write("\n");
+            }
+            writeStream.write("};\n\n");
+            fileInfo.push({
+              uri: '/' + relativeFilePath.replace(sep, '/'),
+              mimeType,
+              variable,
+              size
+            });
+          };
+
+          const writeFiles = () => {
+            // process static files
+            const buildPath = compilation.options.output.path;
+            for (const filePath of getFilesSync(buildPath)) {
+              const readStream = readFileSync(filePath);
+              const relativeFilePath = relative(buildPath, filePath);
+              writeFile(relativeFilePath, readStream);
+            }
+            // process assets
+            const { assets } = compilation;
+            Object.keys(assets).forEach((relativeFilePath) => {
+              writeFile(relativeFilePath, coherseToBuffer(assets[relativeFilePath].source()));
+            });
+          }
+
+          const generateWWWClass = () => {
+            return `typedef std::function<void(const String& uri, const String& contentType, const uint8_t * content, size_t len)> RouteRegistrationHandler;          
+
+class WWWData {
+${indent}public:
+${indent.repeat(2)}static void registerRoutes(RouteRegistrationHandler handler) {
+${fileInfo.map(file => `${indent.repeat(3)}handler("${file.uri}", "${file.mimeType}", ${file.variable}, ${file.size});`).join('\n')}
+${indent.repeat(2)}}
+};
+`;
+          }
+
+          const writeWWWClass = () => {
+            writeStream.write(generateWWWClass());
+          }
+
+          writeIncludes();
+          writeFiles();
+          writeWWWClass();
+
+          writeStream.on('finish', () => {
+            callback();
+          });
+        } finally {
+          writeStream.end();
+        }
+      }
+    );
+  }
+}
+
+module.exports = ProgmemGenerator;

interface/public/css/roboto.css

@@ -3,20 +3,20 @@
   font-family: 'Roboto';
   font-style: normal;
   font-weight: 300;
-  src: local('Roboto Light'), local('Roboto-Light'), url(../fonts/ro-li.w2) format('woff2');
+  src: local('Roboto Light'), local('Roboto-Light'), url(../fonts/li.woff2) format('woff2');
   unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2212, U+2215;
 }
 @font-face {
   font-family: 'Roboto';
   font-style: normal;
   font-weight: 400;
-  src: local('Roboto'), local('Roboto-Regular'), url(../fonts/ro-re.w2) format('woff2');
+  src: local('Roboto'), local('Roboto-Regular'), url(../fonts/re.woff2) format('woff2');
   unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2212, U+2215;
 }
 @font-face {
   font-family: 'Roboto';
   font-style: normal;
   font-weight: 500;
-  src: local('Roboto Medium'), local('Roboto-Medium'), url(../fonts/ro-me.w2) format('woff2');
+  src: local('Roboto Medium'), local('Roboto-Medium'), url(../fonts/me.woff2) format('woff2');
   unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2212, U+2215;
 }