/* * D3-Centraliser - A REST API based database management script * Licence: GPL-3 * Copyright: Jesper Bergman (jesperbe@dsv.su.se) * */ /* Make sure the CORS support works. */ const corsMiddleware = require('restify-cors-middleware'); const errs = require('restify-errors'); const restify = require("restify"); const sqlite3 = require('sqlite3').verbose(); const os = require('os'); const crypto = require('crypto'); const performance = require('performance-now'); const { spawn } = require('child_process'); const fs = require('fs'); /* Server config. */ const server = restify.createServer(); server.use(restify.plugins.acceptParser(server.acceptable)); server.use(restify.plugins.queryParser()); server.use(restify.plugins.bodyParser()); /* Ugly hack: allow request from everywhere. */ const cors = corsMiddleware({ preflightMaxAge: 5, origins: ['*'], allowHeaders: ['*'], exposeHeaders: ['*', 'Content-Type: text/html; charset=UTF-8'] }); /* Activate CORS settings */ server.pre(cors.preflight); server.use(cors.actual); /* Allow CORS from everywhere. Does not work... restify.defaultResponseHeaders = data => { this.header('Access-Control-Allow-Origin', '*'); this.header('Access-Control-Allowed-Methods', 'GET, POST, OPTIONS, DELETE'); this.header('Access-Control-Allowed-Headers', '*'); }*/ /* Database connection. */ const databaseConnection = new sqlite3.Database("annotations.db", (err) => { if(err){console.error( "nej nej", err.message);} }); /* Receive POST for adding stuff to the database. */ server.post('/api/:command', function (req, res, next){ res.setHeader('Access-Control-Allow-Origin', '*'); let feedback = req.body; /* Try - catch after database insertion success/failure */ console.log("Client REQ data: ", req.getQuery(), " . And other than that: ", feedback); let verr_hash = addToDatabase(feedback, req.getQuery()); res.send(200, verr_hash); verr_hash = null; return next(); }); /* REST API get() stuff from the database. */ server.get('/api/:categories', function (req, res, next){ res.setHeader('Access-Control-Allow-Origin', '*'); let feedback = req.body; let categories = databaseConnection.all("SELECT DISTINCT category FROM categories;", function(err,rows){res.send(200, rows)}); //res.send(200, categories); console.log("Received category") return next(); }); /* Main attraction: insert new categories and other data into the DB. Change name to addCategoriesToDatabase*/ function addToDatabase(input, uid){ let inputType = null; let uuid = uid.split("="); uuid = uuid[1]; let sqlUUID = "SELECT uuid FROM users WHERE uuid=\'" + uuid + "\'"; /* Verify message intergrity. */ let verification_hashsum = calculateSHA256(input); console.log("UUID: ", sqlUUID, " Message verification hash sum: ", verification_hashsum); /* Prepare insertion. */ databaseConnection.serialize(function(){ databaseConnection.prepare(sqlUUID); }); for(var i in input){ inputType = i; try{ let com = JSON.parse(inputType) console.log("Command: " , inputType, com.username, com.fullname); } catch{ console.log("This is not a username.") } } if(JSON.parse(inputType).username){ console.log("NOW WE TALK"); /* Insert username and UUID into DB. databaseConnection.serialize(function(){ let query = databaseConnection.prepare("INSERT OR IGNORE INTO users (username) VALUES(?,?)"); query.run("Andersson", uuid); query.finalize(); });*/ } if(inputType == "webpage"){ /* Open up a connection. */ databaseConnection.serialize(function(){ /* Prepare queries for each table. */ let webpageQuery = databaseConnection.prepare("REPLACE INTO webpage (url, domain, sha256, md5, timestamp, uuid) VALUES(?,?,?,?,?,?)"); let categoryQuery = databaseConnection.prepare("REPLACE INTO categories (category, sha256, uuid) VALUES(?,?,?)"); let noteQuery = databaseConnection.prepare("REPLACE INTO notes (note, sha256, uuid) VALUES(?,?,?)"); let highlightedTextQuery = databaseConnection.prepare("REPLACE INTO highlightedText(highlightedText, sha256, uuid) VALUES(?,?,?)"); /* Extract key/values from receivd JSON objects.*/ let firstKey = Object.keys(input); let firstValue = Object.values(input); let secondKey = Object.keys(firstValue[0]); let secondValue = Object.values(firstValue[0]); let webpage = secondValue[0]; let url = webpage.url; webpageQuery.run( webpage.url, webpage.domain, webpage.sha256, webpage.md5, webpage.timestamp, uuid, ); /* Call D3-Collector and time*/ let date = new Date(); let t1 = performance(); let t2 = null; let h = date.getHours(); let m = "0" + date.getMinutes(); let s = "0" + date.getSeconds(); let ss = "0" + date.getMilliseconds(); let formattedTime = h + ':' + m.substr(-2) + ':' + s.substr(-2) + '.' + ss.substr(-2); let ok = archive_webpage(webpage.url); if(ok != null){ t2 = performance(); } let sync_time = t2-t1; console.log("Fetched .onion page in: ", sync_time, " At: ", formattedTime) log(("Fetched .onion page " + webpage.url + " in: " + sync_time + " At: " + formattedTime+"\n\n")) /* Reset */ t1 = null; t1 = null; sync_time = null; /* Parse arrays and refer to the correct SHA256. */ if(webpage.category != "N/A"){ for(var cat in webpage.category){ categoryQuery.run(webpage.category[cat], webpage.sha256, uuid); } } for(var note in webpage.notes){ console.log("|- looping through notes: ", webpage.notes[note]); noteQuery.run(webpage.notes[note], webpage.sha256, uuid); } for(var hlt in webpage.text){ highlightedTextQuery.run(webpage.text[hlt], webpage.sha256, uuid); } /* Write to database. */ t1 = performance(); webpageQuery.finalize(); t2 = performance(); sync_time = t2 - t1; //t1 = performance(); //webpageQuery.finalize(); //2 = performance(); //sync_time = t2 - t1; console.log("DB web page insert time: ", sync_time); t1 = performance(); categoryQuery.finalize(); t2 = performance(); sync_time = t2-t1; console.log("DB category insert time: ", sync_time); t1 = performance(); noteQuery.finalize(); t2 = performance(); sync_time = t2-t1; console.log("DB note insert time: ", sync_time); t1 = performance(); highlightedTextQuery.finalize(); t2 = performance(); sync_time = t2-t1; console.log("DB highlightedText page insert time: ", sync_time); }); } if(inputType == "category"){ databaseConnection.serialize(function(){ let query = databaseConnection.prepare("INSERT INTO categories (category,sha256, uuid) VALUES(?,?,?)"); let stKey = Object.keys(input); let stValue = Object.values(input); let ndValue = Object.values(stValue[0]); console.log("Only categories being added: ", ndValue[0].category); let categories = ndValue[0].category; let sha256 = ndValue[0].sha256; if(categories != "N/A"){ for(var y in categories){ console.log("|-- ", categories[y]); query.run(categories[y], sha256, uuid); } } query.finalize(); }); } return verification_hashsum; verification_hashsum = null; } /* Invoke D3-Collector to archive the web page the URLs are pointing to. */ function archive_webpage(url){ let success = false; const spawn = require("child_process").spawn; const spawnedShell = spawn('sh',["../D3-Collector/start-d3-collector.sh"]); console.log("Spawning .sh to get .onion sites.", url); // Success or not? spawnedShell.stdout.on('data', (data) => { console.log("Data back from D3-Collector: ", data) if(data != null){ success = true; } }); return success; } /* Close connection */ function closeDatabaseConnection(){ databaseConnection.close((err) =>{ if(err){ console.error(err.message); } }); } /* Calculate SHA256 of any input. */ function calculateSHA256(json_candidate){ const hash = crypto.createHash('sha256'); let str = JSON.stringify(json_candidate); hash.update(str); return hash.copy().digest('hex'); hash.update(""); str = ""; console.log("Calculating hash sums...") } function log(log_item){ fs.writeFile('d3-centraliser.log', log_item, function (err) { if (err) return console.log(err); }); } /* Start up the server */ server.listen(8080, "10.1.100.17" , function (){ console.log('%s listening at %s', server.name, server.url); console.log("And other sufttt."); });