gaschz/trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-03-01 14:22:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

#98, sync setup now doesn't copy the whole DB file, but sets up minimal database and starts off sync

Browse Source
This commit is contained in:
azivner 2018-07-23 21:15:32 +02:00
parent a06618d851
commit 1fe7c62f5a
12 changed files with 185 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db/migrations/0102__fix_sync_entityIds.sql Normal file
View File

@ -0,0 +1,2 @@
DELETE FROM sync WHERE entityName = 'note_tree';
DELETE FROM sync WHERE entityName = 'attributes';

1
package.json
View File

@ -61,7 +61,6 @@
"simple-node-logger": "^0.93.37",
"sqlite": "^2.9.2",
"tar-stream": "^1.6.1",
"tmp-promise": "^1.0.5",
"unescape": "^1.0.1",
"ws": "^5.2.1",
"xml2js": "^0.4.19"

27
src/public/javascripts/setup.js
View File

@ -34,7 +34,7 @@ function SetupModel() {
this.setupSyncFromDesktop(false);
};
this.finish = () => {
this.finish = async () => {
if (this.setupNewDocument()) {
const username = this.username();
const password1 = this.password1();
@ -84,20 +84,33 @@ function SetupModel() {
}
// not using server.js because it loads too many dependencies
$.post('/api/setup/sync-from-server', {
const resp = await $.post('/api/setup/sync-from-server', {
serverAddress: serverAddress,
username: username,
password: password
}).then(() => {
window.location.replace("/");
}).catch((err) => {
alert("Error, see dev console for details.");
console.error(err);
});
if (resp.result === 'success') {
this.step('sync-in-progress');
checkOutstandingSyncs();
setInterval(checkOutstandingSyncs, 1000);
}
else {
showAlert('Sync setup failed: ', resp.error);
}
}
};
}
async function checkOutstandingSyncs() {
const stats = await $.get('/api/sync/stats');
const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;
$("#outstanding-syncs").html(totalOutstandingSyncs);
}
function showAlert(message) {
$("#alert").html(message);
$("#alert").show();

88
src/routes/api/setup.js
View File

@ -2,14 +2,10 @@
const sqlInit = require('../../services/sql_init');
const sql = require('../../services/sql');
const cls = require('../../services/cls');
const tmp = require('tmp-promise');
const http = require('http');
const fs = require('fs');
const rp = require('request-promise');
const Option = require('../../entities/option');
const syncService = require('../../services/sync');
const log = require('../../services/log');
const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH;
const sourceIdService = require('../../services/source_id');
const url = require('url');
async function setupNewDocument(req) {
const { username, password } = req.body;
@ -20,52 +16,44 @@ async function setupNewDocument(req) {
async function setupSyncFromServer(req) {
const { serverAddress, username, password } = req.body;
const tempFile = await tmp.file();
try {
log.info("Getting document options from sync server.");
await new Promise((resolve, reject) => {
const file = fs.createWriteStream(tempFile.path);
const parsedAddress = url.parse(serverAddress);
// response is expected to contain documentId and documentSecret options
const options = await rp.get({
uri: serverAddress + '/api/sync/document',
auth: {
'user': username,
'pass': password
},
json: true
});
const options = {
method: 'GET',
protocol: parsedAddress.protocol,
host: parsedAddress.hostname,
port: parsedAddress.port,
path: '/api/sync/document',
auth: username + ':' + password
log.info("Creating database for sync");
await sql.transactional(async () => {
await sqlInit.createDatabaseForSync(serverAddress);
for (const opt of options) {
await new Option(opt).save();
}
});
log.info("Triggering sync.");
// it's ok to not wait for it here
syncService.sync();
return { result: 'success' };
}
catch (e) {
log.error("Sync failed: " + e.message);
return {
result: 'failure',
error: e.message
};
log.info("Getting document from: " + serverAddress);
http.request(options, function(response) {
response.pipe(file);
file.on('finish', function() {
log.info("Document download finished, closing & renaming.");
file.close(() => { // close() is async, call after close completes.
fs.rename(tempFile.path, DOCUMENT_PATH, async () => {
cls.reset();
await sqlInit.initDbConnection();
// we need to generate new source ID for this instance, otherwise it will
// match the original server one
await sql.transactional(async () => {
await sourceIdService.generateSourceId();
});
resolve();
});
});
});
}).on('error', function(err) { // Handle errors
fs.unlink(tempFile.path); // Delete the file async. (But we don't check the result)
reject(err.message);
log.error(err.message);
}).end();
});
}
}
module.exports = {

23
src/routes/api/sync.js
View File

@ -7,7 +7,7 @@ const sql = require('../../services/sql');
const optionService = require('../../services/options');
const contentHashService = require('../../services/content_hash');
const log = require('../../services/log');
const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH;
const repository = require('../../services/repository');
async function testSync() {
try {
@ -23,6 +23,10 @@ async function testSync() {
}
}
async function getStats() {
return syncService.stats;
}
async function checkSync() {
return {
hashes: await contentHashService.getHashes(),
@ -75,7 +79,10 @@ async function getChanged(req) {
const syncs = await sql.getRows("SELECT * FROM sync WHERE id > ? LIMIT 1000", [lastSyncId]);
return await syncService.getSyncRecords(syncs);
return {
syncs: await syncService.getSyncRecords(syncs),
maxSyncId: await sql.getValue('SELECT MAX(id) FROM sync')
};
}
async function update(req) {
@ -87,10 +94,13 @@ async function update(req) {
}
}
async function getDocument(req, resp) {
log.info("Serving document.");
async function getDocument() {
log.info("Serving document options.");
resp.sendFile(DOCUMENT_PATH);
return [
await repository.getOption('documentId'),
await repository.getOption('documentSecret')
];
}
module.exports = {
@ -102,5 +112,6 @@ module.exports = {
forceNoteSync,
getChanged,
update,
getDocument
getDocument,
getStats
};

3
src/routes/routes.js
View File

@ -156,7 +156,8 @@ function register(app) {
apiRoute(POST, '/api/sync/force-note-sync/:noteId', syncApiRoute.forceNoteSync);
apiRoute(GET, '/api/sync/changed', syncApiRoute.getChanged);
apiRoute(PUT, '/api/sync/update', syncApiRoute.update);
route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument);
route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument, apiResultHandler);
route(GET, '/api/sync/stats', [], syncApiRoute.getStats, apiResultHandler);
apiRoute(GET, '/api/event-log', eventLogRoute.getEventLog);

2
src/services/app_info.js
View File

@ -3,7 +3,7 @@
const build = require('./build');
const packageJson = require('../../package');
const APP_DB_VERSION = 101;
const APP_DB_VERSION = 102;
const SYNC_VERSION = 1;
module.exports = {

11
src/services/note_cache.js
View File

@ -5,6 +5,7 @@ const repository = require('./repository');
const protectedSessionService = require('./protected_session');
const utils = require('./utils');
let loaded = false;
let noteTitles;
let protectedNoteTitles;
let noteIds;
@ -34,6 +35,8 @@ async function load() {
for (const noteId of hiddenLabels) {
archived[noteId] = true;
}
loaded = true;
}
function findNotes(query) {
@ -226,6 +229,10 @@ function getNotePath(noteId) {
}
eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entityId}) => {
if (!loaded) {
return;
}
if (entityName === 'notes') {
const note = await repository.getNote(entityId);
@ -277,6 +284,10 @@ eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entityId
});
eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, async () => {
if (!loaded) {
return;
}
protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`);
for (const noteId in protectedNoteTitles) {

35
src/services/options_init.js
View File

@ -5,21 +5,14 @@ const appInfo = require('./app_info');
const utils = require('./utils');
const dateUtils = require('./date_utils');
async function initOptions(startNotePath, username, password) {
async function initDocumentOptions() {
await optionService.createOption('documentId', utils.randomSecureToken(16), false);
await optionService.createOption('documentSecret', utils.randomSecureToken(16), false);
}
await optionService.createOption('startNotePath', startNotePath, false);
await optionService.createOption('protectedSessionTimeout', 600, true);
await optionService.createOption('noteRevisionSnapshotTimeInterval', 600, true);
await optionService.createOption('lastBackupDate', dateUtils.nowDate(), false);
await optionService.createOption('dbVersion', appInfo.dbVersion, false);
await optionService.createOption('lastSyncedPull', appInfo.dbVersion, false);
await optionService.createOption('lastSyncedPush', 0, false);
await optionService.createOption('zoomFactor', 1.0, false);
await optionService.createOption('theme', 'white', false);
async function initSyncedOptions(username, password) {
await optionService.createOption('protectedSessionTimeout', 600);
await optionService.createOption('noteRevisionSnapshotTimeInterval', 600);
await optionService.createOption('username', username);
@ -34,12 +27,26 @@ async function initOptions(startNotePath, username, password) {
await optionService.createOption('encryptedDataKeyIv', '');
await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16));
}
await optionService.createOption('syncServerHost', '', false);
async function initNotSyncedOptions(startNotePath = '', syncServerHost = '') {
await optionService.createOption('startNotePath', startNotePath, false);
await optionService.createOption('lastBackupDate', dateUtils.nowDate(), false);
await optionService.createOption('dbVersion', appInfo.dbVersion, false);
await optionService.createOption('lastSyncedPull', appInfo.dbVersion, false);
await optionService.createOption('lastSyncedPush', 0, false);
await optionService.createOption('zoomFactor', 1.0, false);
await optionService.createOption('theme', 'white', false);
await optionService.createOption('syncServerHost', syncServerHost, false);
await optionService.createOption('syncServerTimeout', 5000, false);
await optionService.createOption('syncProxy', '', false);
}
module.exports = {
initOptions
initDocumentOptions,
initSyncedOptions,
initNotSyncedOptions
};

39
src/services/sql_init.js
View File

@ -12,9 +12,13 @@ async function createConnection() {
}
let dbReadyResolve = null;
const dbReady = new Promise((resolve, reject) => {
const dbReady = new Promise(async (resolve, reject) => {
dbReadyResolve = resolve;
// no need to create new connection now since DB stays the same all the time
const db = await createConnection();
sql.setDbConnection(db);
initDbConnection();
});
@ -26,9 +30,6 @@ async function isDbInitialized() {
async function initDbConnection() {
await cls.init(async () => {
const db = await createConnection();
sql.setDbConnection(db);
await sql.execute("PRAGMA foreign_keys = ON");
if (!await isDbInitialized()) {
@ -45,12 +46,12 @@ async function initDbConnection() {
}
log.info("DB ready.");
dbReadyResolve(db);
dbReadyResolve();
});
}
async function createInitialDatabase(username, password) {
log.info("Connected to db, but schema doesn't exist. Initializing schema ...");
log.info("Creating initial database ...");
const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
const notesSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_notes.sql', 'UTF-8');
@ -67,15 +68,34 @@ async function createInitialDatabase(username, password) {
const startNoteId = await sql.getValue("SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 ORDER BY notePosition");
await require('./options_init').initOptions(startNoteId, username, password);
const optionsInitService = require('./options_init');
await optionsInitService.initDocumentOptions();
await optionsInitService.initSyncedOptions(username, password);
await optionsInitService.initNotSyncedOptions(startNoteId);
await require('./sync_table').fillAllSyncRows();
});
log.info("Schema and initial content generated. Waiting for user to enter username/password to finish setup.");
log.info("Schema and initial content generated.");
await initDbConnection();
}
async function createDatabaseForSync(syncServerHost) {
log.info("Creating database for sync with server ...");
const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
await sql.transactional(async () => {
await sql.executeScript(schema);
await require('./options_init').initNotSyncedOptions('', syncServerHost);
});
log.info("Schema and not synced options generated.");
}
async function isDbUpToDate() {
const dbVersion = parseInt(await sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'"));
@ -93,5 +113,6 @@ module.exports = {
isDbInitialized,
initDbConnection,
isDbUpToDate,
createInitialDatabase
createInitialDatabase,
createDatabaseForSync
};

44
src/services/sync.js
View File

@ -18,6 +18,11 @@ const cls = require('./cls');
let proxyToggle = true;
const stats = {
outstandingPushes: 0,
outstandingPulls: 0
};
async function sync() {
try {
await syncMutexService.doExclusively(async () => {
@ -82,21 +87,33 @@ async function login() {
}
async function pullSync(syncContext) {
const changesUri = '/api/sync/changed?lastSyncId=' + await getLastSyncedPull();
while (true) {
const lastSyncedPull = await getLastSyncedPull();
const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
const rows = await syncRequest(syncContext, 'GET', changesUri);
const resp = await syncRequest(syncContext, 'GET', changesUri);
stats.outstandingPulls = resp.maxSyncId - lastSyncedPull;
log.info("Pulled " + rows.length + " changes from " + changesUri);
const rows = resp.syncs;
for (const {sync, entity} of rows) {
if (sourceIdService.isLocalSourceId(sync.sourceId)) {
log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
}
else {
await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
if (rows.length === 0) {
break;
}
await setLastSyncedPull(sync.id);
log.info("Pulled " + rows.length + " changes from " + changesUri);
for (const {sync, entity} of rows) {
if (sourceIdService.isLocalSourceId(sync.sourceId)) {
log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
}
else {
await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
}
stats.outstandingPulls = resp.maxSyncId - sync.id;
await setLastSyncedPull(sync.id);
}
}
log.info("Finished pull");
@ -127,6 +144,8 @@ async function pushSync(syncContext) {
if (filteredSyncs.length === 0) {
log.info("Nothing to push");
stats.outstandingPushes = 0;
await setLastSyncedPush(lastSyncedPush);
break;
@ -144,6 +163,8 @@ async function pushSync(syncContext) {
lastSyncedPush = syncRecords[syncRecords.length - 1].sync.id;
await setLastSyncedPush(lastSyncedPush);
stats.outstandingPushes = await sql.getValue(`SELECT MAX(id) FROM sync`) - lastSyncedPush;
}
}
@ -290,5 +311,6 @@ sqlInit.dbReady.then(async () => {
module.exports = {
sync,
login,
getSyncRecords
getSyncRecords,
stats
};

10
src/views/setup.ejs
View File

@ -86,6 +86,16 @@
<button type="button" data-bind="click: finish" class="btn btn-primary">Finish setup</button>
</div>
<div data-bind="visible: step() == 'sync-in-progress'">
<h2>Sync in progress</h2>
<div class="alert alert-success">Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.</div>
<div>
Outstanding sync items: <strong id="outstanding-syncs">N/A</strong>
</div>
</div>
</div>
<script type="text/javascript">