Add pagination support for completed items and projects

public/js/client.js

@@ -28,4 +28,16 @@ window.onload = function() {
     ).innerText = persistentBackupUrl;
     window.location.replace(persistentBackupUrl);
   }
+
+  updateArchived(this);
 };
+
+function updateArchived(sender) { 
+  var archived = document.getElementById("archivedCbox");
+  if (document.getElementById("formatJSON").checked) {
+    archived.disabled = false;
+  } else {
+    archived.disabled = true;
+    archived.checked = false;
+  }
+}

src/app.js

@@ -26,6 +26,8 @@ const oauth2 = require("simple-oauth2").create({
 });
 
 const IS_PRODUCTION = process.env.NODE_ENV === "production";
+const COMPL_MAX_PAGE_SIZE = 200;
+const FORMAT_SUFFIX_INCLUDE_ARCHIVED = "_all";
 
 const app = express();
 const subdirectory = "/todoist-export";
@@ -77,7 +79,10 @@ const renderErrorPage = (res, message, error) => {
 };
 
 app.post(`${subdirectory}/auth`, (req, res) => {
-  const format = req.body.format || "json"; // csv vs. json
+  var format = req.body.format || "json"; // csv vs. json
+  if (req.body.archived) {
+      format += FORMAT_SUFFIX_INCLUDE_ARCHIVED;
+  }
 
   res.redirect(
     oauth2.authorizationCode.authorizeURL({
@@ -157,6 +162,19 @@ const convertUserNames = syncData => {
   }));
 };
 
+const fetchCompleted = async function(token, offset = 0) {
+  const page = await callApi("completed/get_all", { token: token, limit: COMPL_MAX_PAGE_SIZE, offset: offset })
+  if (page.items.length == COMPL_MAX_PAGE_SIZE || Object.keys(page.projects).length == COMPL_MAX_PAGE_SIZE) {
+    const remainder = await fetchCompleted(token, offset + COMPL_MAX_PAGE_SIZE);
+    return {
+      items: page.items.concat(remainder.items),
+      projects: Object.assign({}, page.projects, remainder.projects),
+    };
+  } else {
+    return page;
+  }
+};
+
 const exportData = async (res, token, format = "csv") => {
   const syncData = await callApi("sync", {
     token: token,
@@ -169,10 +187,13 @@ const exportData = async (res, token, format = "csv") => {
   }
 
   // Fetch completed tasks (premium-only)
-  const completedData = syncData.user.is_premium
-    ? await callApi("completed/get_all", { token: token })
-    : undefined;
-  syncData.completed = completedData; // add completed tasks
+  if (format.includes(FORMAT_SUFFIX_INCLUDE_ARCHIVED)) {
+      if (!syncData.user.is_premium) {
+          return renderErrorPage(res, "Must be Todoist Premium to export archived items.");
+      }
+      format = format.replace(FORMAT_SUFFIX_INCLUDE_ARCHIVED, '');
+      syncData.completed = await fetchCompleted(token);
+  }
 
   if (format === "json") {
     res.attachment("todoist.json");

src/views/index.pug

@@ -19,12 +19,20 @@ block content
           div.radios
             div.radio
               label
-                input(type="radio", name="format", id="formatCSV", value="csv", checked="checked")
+                input(type="radio", name="format", id="formatCSV", value="csv", checked="checked" onchange="updateArchived(this)")
                 | CSV (only tasks)
             div.radio
               label
-                input(type="radio", name="format", id="formatJSON", value="json")
+                input(type="radio", name="format", id="formatJSON", value="json", onchange="updateArchived(this)")
                 | JSON (all data)
+        div.form-group#archived
+          label(for="archived")
+            strong Archived:
+          div.checkbox
+            label
+              input(type="checkbox", name="archived", id="archivedCbox")
+              | Export all
+              | (requires Todoist Premium and may take several minutes)
 
         div.text-center
           button#submit(type="submit")
@@ -41,6 +49,7 @@ block content
     h2 Changelog
 
     ul
+      li <strong>2021-02-15</strong>: Option to export all archived tasks and projects.
       li <strong>2020-02-02</strong>: The CSV format contains translated label, project and user names (instead of ID numbers).
       li <strong>2019-06-18</strong>: Updated to <a href="https://developer.todoist.com/sync/v8/#migration-guide" target="_blank" rel="noopener">API v8</a>. The output file schema has changed a little bit.
       li <strong>2016-09-01</strong>: Updated to API v7.