From 66064f7a942bb6f0d820506c2f5da46ca33e8c97 Mon Sep 17 00:00:00 2001
From: azivner <adam.zivner@gmail.com>
Date: Mon, 26 Feb 2018 20:47:34 -0500
Subject: [PATCH] Script API changes, finished porting reddit plugin, reddit
 importer tar file

---
 src/plugins/reddit.js           | 144 --------------------------------
 src/routes/api/export.js        |  25 +++---
 src/scripts/Reddit Importer.tar | Bin 0 -> 10752 bytes
 src/services/attributes.js      |   3 +-
 src/services/notes.js           |   3 +
 src/services/script_context.js  |  11 ++-
 src/services/utils.js           |   8 +-
 7 files changed, 35 insertions(+), 159 deletions(-)
 delete mode 100644 src/plugins/reddit.js
 create mode 100644 src/scripts/Reddit Importer.tar

diff --git a/src/plugins/reddit.js b/src/plugins/reddit.js
deleted file mode 100644
index 18237eb88..000000000
--- a/src/plugins/reddit.js
+++ /dev/null
@@ -1,144 +0,0 @@
-"use strict";
-
-const sql = require('../services/sql');
-const notes = require('../services/notes');
-const axios = require('axios');
-const log = require('../services/log');
-const utils = require('../services/utils');
-const unescape = require('unescape');
-const attributes = require('../services/attributes');
-const sync_mutex = require('../services/sync_mutex');
-const config = require('../services/config');
-const date_notes = require('../services/date_notes');
-
-// "reddit" date note is subnote of date note which contains all reddit comments from that date
-const REDDIT_DATE_ATTRIBUTE = 'reddit_date_note';
-
-async function createNote(parentNoteId, noteTitle, noteText) {
-    return (await notes.createNewNote(parentNoteId, {
-        title: noteTitle,
-        content: noteText,
-        target: 'into',
-        isProtected: false
-    })).noteId;
-}
-
-function redditId(kind, id) {
-    return kind + "_" + id;
-}
-
-async function getDateNoteIdForReddit(dateTimeStr, rootNoteId) {
-    const dateStr = dateTimeStr.substr(0, 10);
-
-    let redditDateNoteId = await attributes.getNoteIdWithAttribute(REDDIT_DATE_ATTRIBUTE, dateStr);
-
-    if (!redditDateNoteId) {
-        const dateNoteId = await date_notes.getDateNoteId(dateTimeStr, rootNoteId);
-
-        redditDateNoteId = await createNote(dateNoteId, "Reddit");
-
-        await attributes.createAttribute(redditDateNoteId, REDDIT_DATE_ATTRIBUTE, dateStr);
-        await attributes.createAttribute(redditDateNoteId, "hide_in_autocomplete");
-    }
-
-    return redditDateNoteId;
-}
-
-async function importComments(rootNoteId, accountName, afterId = null) {
-    let url = `https://www.reddit.com/user/${accountName}.json`;
-
-    if (afterId) {
-        url += "?after=" + afterId;
-    }
-
-    const response = await axios.get(url);
-    const listing = response.data;
-
-    if (listing.kind !== 'Listing') {
-        log.info(`Reddit: Unknown object kind ${listing.kind}`);
-        return;
-    }
-
-    const children = listing.data.children;
-
-    let importedComments = 0;
-
-    for (const child of children) {
-        const comment = child.data;
-
-        let commentNoteId = await attributes.getNoteIdWithAttribute('reddit_id', redditId(child.kind, comment.id));
-
-        if (commentNoteId) {
-            continue;
-        }
-
-        const dateTimeStr = utils.dateStr(new Date(comment.created_utc * 1000));
-
-        const permaLink = 'https://reddit.com' + comment.permalink;
-
-        const noteText =
-`<p><a href="${permaLink}">${permaLink}</a></p>
-<p>author: <a href="https://reddit.com/u/${comment.author}">${comment.author}</a>, 
-subreddit: <a href="https://reddit.com/r/${comment.subreddit}">${comment.subreddit}</a>, 
-karma: ${comment.score}, created at ${dateTimeStr}</p><p></p>`
-            + unescape(comment.body_html);
-
-        let parentNoteId = await getDateNoteIdForReddit(dateTimeStr, rootNoteId);
-
-        await sql.doInTransaction(async () => {
-            commentNoteId = await createNote(parentNoteId, comment.link_title, noteText);
-
-            log.info("Reddit: Imported comment to note " + commentNoteId);
-            importedComments++;
-
-            await attributes.createAttribute(commentNoteId, "reddit_kind", child.kind);
-            await attributes.createAttribute(commentNoteId, "reddit_id", redditId(child.kind, comment.id));
-            await attributes.createAttribute(commentNoteId, "reddit_created_utc", comment.created_utc);
-        });
-    }
-
-    // if there have been no imported comments on this page, there shouldn't be any to import
-    // on the next page since those are older
-    if (listing.data.after && importedComments > 0) {
-        importedComments += await importComments(rootNoteId, accountName, listing.data.after);
-    }
-
-    return importedComments;
-}
-
-let redditAccounts = [];
-
-async function runImport() {
-    const rootNoteId = await date_notes.getRootCalendarNoteId();
-
-    // technically mutex shouldn't be necessary but we want to avoid doing potentially expensive import
-    // concurrently with sync
-    await sync_mutex.doExclusively(async () => {
-        let importedComments = 0;
-
-        for (const account of redditAccounts) {
-            importedComments += await importComments(rootNoteId, account);
-        }
-
-        log.info(`Reddit: Imported ${importedComments} comments.`);
-    });
-}
-
-sql.dbReady.then(async () => {
-    if (!config['Reddit'] || config['Reddit']['enabled'] !== true) {
-        return;
-    }
-
-    const redditAccountsStr = config['Reddit']['accounts'];
-
-    if (!redditAccountsStr) {
-        log.info("Reddit: No reddit accounts defined in option 'reddit_accounts'");
-    }
-
-    redditAccounts = redditAccountsStr.split(",").map(s => s.trim());
-
-    const pollingIntervalInSeconds = config['Reddit']['pollingIntervalInSeconds'] || (4 * 3600);
-
-    setInterval(runImport, pollingIntervalInSeconds * 1000);
-    setTimeout(runImport, 10000); // 10 seconds after startup - intentionally after initial sync
-});
diff --git a/src/routes/api/export.js b/src/routes/api/export.js
index 344a8da76..47b241361 100644
--- a/src/routes/api/export.js
+++ b/src/routes/api/export.js
@@ -35,17 +35,20 @@ async function exportNote(noteTreeId, directory, pack) {
         return;
     }
 
-    const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content;
-
-    const childFileName = directory + sanitize(note.title);
-
-    console.log(childFileName);
-
-    pack.entry({ name: childFileName + ".dat", size: content.length }, content);
-
     const metadata = await getMetadata(note);
 
-    pack.entry({ name: childFileName + ".meta", size: metadata.length }, metadata);
+    if ('exclude_from_export' in metadata.attributes) {
+        return;
+    }
+
+    const metadataJson = JSON.stringify(metadata, null, '\t');
+    const childFileName = directory + sanitize(note.title);
+
+    pack.entry({ name: childFileName + ".meta", size: metadataJson.length }, metadataJson);
+
+    const content = note.type === 'text' ? html.prettyPrint(note.content, {indent_size: 2}) : note.content;
+
+    pack.entry({ name: childFileName + ".dat", size: content.length }, content);
 
     const children = await sql.getRows("SELECT * FROM note_tree WHERE parentNoteId = ? AND isDeleted = 0", [note.noteId]);
 
@@ -59,14 +62,12 @@ async function exportNote(noteTreeId, directory, pack) {
 }
 
 async function getMetadata(note) {
-    const meta = {
+    return {
         title: note.title,
         type: note.type,
         mime: note.mime,
         attributes: await attributes.getNoteAttributeMap(note.noteId)
     };
-
-    return JSON.stringify(meta, null, '\t')
 }
 
 module.exports = router;
\ No newline at end of file
diff --git a/src/scripts/Reddit Importer.tar b/src/scripts/Reddit Importer.tar
new file mode 100644
index 0000000000000000000000000000000000000000..b8145e9825b9ce1194f939cd6a1063a9d77a1a16
GIT binary patch
literal 10752
zcmeHNTW=dT7S3z*D^3SNEhmaD@g;y9dEwewR7KJUS4AHdg-mHEYvK_p%y6O_ss8(Z
zhvaB3^2P}wv?x{sB$kFe7apF=$3tDqFpLZh7fYQOnY0(uh$lOKy4~*S@i9HYUp)1{
z-EQyoYw8`nIzH(g9i8-!soOj1;?<Mxj`etuU6vY=K(z-4zv5+mabH3DcTWA(cp8|<
z#4_kpa9wqVpv8mtOFK5zp|md+(ZaqE%Viu*g^9H4d=MW+I!&Ubu~URGNi@lfOu-cr
zo(4&###)Wp)+SrBtf299-B|yfOW(_e1!`N~4u#pdT@V_di~rt9_oNp8uU_{~_VK^-
zjd*}z=Xryk(@4`)t66lLC6dfMO36iYi}X?|n$Dv*B&Cg{sfc44QsAb)OvPe}+|i;;
zP4!|Sm7&O_%*^m-p~4DdNvSj@QnzWF#a?N3U}&b3TWQE>N>`DY&MA;eOduuTcP7bX
zn?{9B1nm|gxq}uMMq(kYsFaMzCmDK)fvKc>ogH9Ri4@^|i>2ZPeL&hLB^{4B3gG4F
z`rXTGTFoOUu!`as1hEz&%@Vtoup)xjl}3v!7BI;JvfK2V;FXkzVtt#kcr2HeiN*pK
zDdF=%L%TWbJj>$v-VQHhnu=SQl9=cW7Inp$O=(SRjn`>3q8T;6OKi@*w7W-t{z;V=
zZEs0x5FU&+%hTJDf~{bqpT$Db_upg53<eyp>fm>S0TvUru~EYkQS<Qd5DA`{C(>kz
zI%}*OjZ`Kd0VS~2Kj}%+4J(L1)^`%denknYg{95ZTTtA;Py1CJmO75%Ekk%y@=?U^
zQ#U_Sk%>fnovDi%Mz+YT2lydJu@EW~@hA~06;m!*KVl1+b>BLeoAg0XAU{(M47qnp
zk%$G<`cG9cF`ep6!P9?ltPjr~h^v3ftLtjX?`)R8LnQb}{_pmVyS4m(f`YNn|9^!K
zRq)?t^LK>&_5km%*urm;@xSALcS!J&_&+^;eN>D8<JYhD=f7{-*Doa&>AjlL4DgMs
zVp07(P`L1QF0b;PPF}+Uf`dp03K=hE6N?Y^tn}WYDF%ZAP=f#+eHGmx8HQI2AWmT{
znMDJb4{&z<_UdXl8ed(E-i|LuqwC=hzmDG0fPm!~q>Nemm_;6(H7r@N20LLGHt!-8
zwkQg7<qMcXJVY-j7zg-23ePMDKo_1;GYKHi6=eMkQN!@xI&l%xWcec;g?}+gixRDk
zn^2bJ7Qz!T396S!+OSt@l4iF>z3!o30Sk9DZjHHEkcyQ6Nc9MeCH)qe`9*%rZa!$N
zmK6xm;Loi2VaQlzURIcAI2`BqDc7LQh8}t|@hopD{#dNiS)i_M&I21#GC;*<rS*I9
z#d@az?#}v_+{tL=y$YN>0{n9rleOWeXg_fG!M!W~f6!L_+*S_t6|;$gc@)ZVq{bpM
z8VDTc9b+3HL51~t^9_K|ety`U9Q3{PNWNKkPK!LKWHgR1jBJM?tf_D=DDM*FQ<lUS
zxtSZYO#7YAYPD*+ZEZutP6h<tdG@KqZ{7ZoYIRdGG<$-VQD)97XXY;k6#UDM4>&~p
z%)*hj93YYD5`il#+b{lzbjpW@CU}<I(D_UprF<3yMNZ7^2T@bf&uZI{<#WivJ4bg=
z*PFuwr>y3U3(-FPs_v9t;ea^#Ag9I!{j*PHhU=S>R=oE77{j0W*;=PP3nw@f4=88E
zO1Ja(CGB~m^0QmM;UMdlrto2~>8#-n*~p(cr~2~zG=Iz<xOsJpvCfr=959tV!EW%o
zY68i#Y7`zIGV=s#HL-?o<r%w;bPAJgoSo&dE5Ysf^v|5@dV7_y)tM|Dh~Jis>n9=c
zmy!hQjXljw6sPSd)mKH%Z7R7U{+>&2yF?L=Gc(00y9dy*rKmH^QYH)WE>d@3e~?E|
z5j_WRcfXppor+`PU2VmVk-}m}@()7;YmS>Y%kwva=82pQf@hyLV%EWV`P-Y0IDgYw
zo?9A_Ag#=G(x(mSwo080(VXk#=3Az<SIoKPj`}DsZbhHx%UL34vLUIkF5Z^7-w9Z-
zPbJzZniy+1iA~n%l^6_`Hge4iX7k|BO~rS3-;rK6yPPSRPQ_CGXcqBiYbQFqAJ6fo
z?s<Mg1>(`y-;Jw(W!bK6a8Z9u(KBc|WY9dM!TAG)Wh?A#_AWS;DjB*88;y(5Vn-X>
zW_-)yD)y~<ZDV`!^Z6R*=Ad>tCm3_?MAN+#&s&>NCyAKgu}rXCE$eJbQOOFpEK!ct
z2NfxbY0i&}>ekde;Rn|?WA*cEtWBqkJA@pi#5K$d=M*&zYiU2P*}1risCo{6@uEm$
zr4$Bij7(_elKXk$qoj#MnNWID19A<}_EWCQx<r?+<w?@H&f<_8q7(EK)jjL(ByW^~
z1WO6n&1r|J(T3zECsGJXFT9F$9Li)v!Kt}YrfebSUSEAtbWYuR=H8kERN(yCja)v8
zVzy<M3}3RHhkbQ{6(Hr}s{l-!P-izlxL*T=Uy4|&P$X`Xo4M0GQi)4d*kdL_9d^!N
zKLN3a)YW9P^rd$3>-THVJ$M6IhIy~|*{8ac^~QhNj9PYccjh7|51ga}BjXyt_w{$w
z1AOHCkN)3r?fidoy1)PV(i8l*&8zhPDx1IW0e+ih{~6SukAFb89W3Lq_&@E{fB$iE
zY=1AZ1KE$V_&@*sN9Q@$Mpx=C-6i6{V_ScB*r%C@Q@|9U4}U@HuVr0Vz}~0Ma3z<{
zQJ~x<vi((v1tH}t#=dQ3I7Sev6|Or0Q20L9;w9Xtnh5O`8YNL2WeduQac_#Je5==|
zo<HPSc5}U=SfiB?YFbIM&9aF^S!ZM<X%;0oSmJiW^4Ypn?9(xr9i8I0N$%iic^WkL
z0L9FTQ5<NU5*{De8^PfHy+;QD{m$lbX3YCI#CM*lzfyMSj-efnH%j5+nL!61Vm&NP
Y8|$;@ok!huKVffyy#@9b*p&tT3n-V$QUCw|

literal 0
HcmV?d00001

diff --git a/src/services/attributes.js b/src/services/attributes.js
index f1f752c0b..c597c766e 100644
--- a/src/services/attributes.js
+++ b/src/services/attributes.js
@@ -9,7 +9,8 @@ const BUILTIN_ATTRIBUTES = [
     'run_on_startup',
     'disable_versioning',
     'calendar_root',
-    'hide_in_autocomplete'
+    'hide_in_autocomplete',
+    'exclude_from_export'
 ];
 
 async function getNoteAttributeMap(noteId) {
diff --git a/src/services/notes.js b/src/services/notes.js
index 38ecf9b37..3f38f1039 100644
--- a/src/services/notes.js
+++ b/src/services/notes.js
@@ -84,6 +84,9 @@ async function createNewNote(parentNoteId, noteOpts, dataKey, sourceId) {
 }
 
 async function createNote(parentNoteId, title, content = "", extraOptions = {}) {
+    if (!parentNoteId) throw new Error("Empty parentNoteId");
+    if (!title) throw new Error("Empty title");
+
     const note = {
         title: title,
         content: extraOptions.json ? JSON.stringify(content, null, '\t') : content,
diff --git a/src/services/script_context.js b/src/services/script_context.js
index 921504ba7..8a5ddcc07 100644
--- a/src/services/script_context.js
+++ b/src/services/script_context.js
@@ -2,15 +2,24 @@ const log = require('./log');
 const protected_session = require('./protected_session');
 const notes = require('./notes');
 const sql = require('./sql');
+const utils = require('./utils');
 const attributes = require('./attributes');
 const date_notes = require('./date_notes');
 const config = require('./config');
 const Repository = require('./repository');
+const axios = require('axios');
 
 function ScriptContext(dataKey) {
     dataKey = protected_session.getDataKey(dataKey);
     const repository = new Repository(dataKey);
 
+    this.axios = axios;
+
+    this.utils = {
+        unescapeHtml: utils.unescapeHtml,
+        isoDateTimeStr: utils.dateStr
+    };
+
     this.getInstanceName = () => config.General ? config.General.instanceName : null;
 
     this.getNoteById = async function(noteId) {
@@ -30,7 +39,7 @@ function ScriptContext(dataKey) {
     this.createNote = async function(parentNoteId, title, content = "", extraOptions = {}) {
         extraOptions.dataKey = dataKey;
 
-        notes.createNote(parentNoteId, title, content, extraOptions);
+        return await notes.createNote(parentNoteId, title, content, extraOptions);
     };
 
     this.createAttribute = attributes.createAttribute;
diff --git a/src/services/utils.js b/src/services/utils.js
index 5596ca933..a6c4e0645 100644
--- a/src/services/utils.js
+++ b/src/services/utils.js
@@ -2,6 +2,7 @@
 
 const crypto = require('crypto');
 const randtoken = require('rand-token').generator({source: 'crypto'});
+const unescape = require('unescape');
 
 function newNoteId() {
     return randomString(12);
@@ -129,6 +130,10 @@ async function stopWatch(what, func) {
     return ret;
 }
 
+function unescapeHtml(str) {
+    return unescape(str);
+}
+
 module.exports = {
     randomSecureToken,
     randomString,
@@ -153,5 +158,6 @@ module.exports = {
     getDateTimeForFile,
     sanitizeSql,
     assertArguments,
-    stopWatch
+    stopWatch,
+    unescapeHtml
 };
\ No newline at end of file