276 lines
8.7 KiB
JavaScript
276 lines
8.7 KiB
JavaScript
/*
|
|
* D3-Centraliser - A REST API based database management script
|
|
* Licence: GPL-3
|
|
* Copyright: Jesper Bergman (jesperbe@dsv.su.se)
|
|
*
|
|
*/
|
|
|
|
/* Make sure the CORS support works. */
|
|
|
|
const corsMiddleware = require('restify-cors-middleware');
|
|
const errs = require('restify-errors');
|
|
const restify = require("restify");
|
|
const sqlite3 = require('sqlite3').verbose();
|
|
const os = require('os');
|
|
const crypto = require('crypto');
|
|
const performance = require('performance-now');
|
|
const { spawn } = require('child_process');
|
|
const fs = require('fs');
|
|
|
|
/* Server config. */
|
|
const server = restify.createServer();
|
|
server.use(restify.plugins.acceptParser(server.acceptable));
|
|
server.use(restify.plugins.queryParser());
|
|
server.use(restify.plugins.bodyParser());
|
|
|
|
/* Ugly hack: allow request from everywhere. */
|
|
const cors = corsMiddleware({
|
|
preflightMaxAge: 5,
|
|
origins: ['*'],
|
|
allowHeaders: ['*'],
|
|
exposeHeaders: ['*', 'Content-Type: text/html; charset=UTF-8']
|
|
|
|
});
|
|
|
|
/* Activate CORS settings */
|
|
server.pre(cors.preflight);
|
|
server.use(cors.actual);
|
|
|
|
/* Allow CORS from everywhere. Does not work...
|
|
restify.defaultResponseHeaders = data => {
|
|
this.header('Access-Control-Allow-Origin', '*');
|
|
this.header('Access-Control-Allowed-Methods', 'GET, POST, OPTIONS, DELETE');
|
|
this.header('Access-Control-Allowed-Headers', '*');
|
|
}*/
|
|
|
|
/* Database connection. */
|
|
const databaseConnection = new sqlite3.Database("annotations.db", (err) => {
|
|
if(err){console.error( "nej nej", err.message);}
|
|
});
|
|
|
|
/* Receive POST for adding stuff to the database. */
|
|
server.post('/api/:command', function (req, res, next){
|
|
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
let feedback = req.body;
|
|
/* Try - catch after database insertion success/failure */
|
|
console.log("Client REQ data: ", req.getQuery(), " . And other than that: ", feedback);
|
|
let verr_hash = addToDatabase(feedback, req.getQuery());
|
|
res.send(200, verr_hash);
|
|
verr_hash = null;
|
|
return next();
|
|
});
|
|
|
|
|
|
/* REST API get() stuff from the database. */
|
|
server.get('/api/:categories', function (req, res, next){
|
|
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
let feedback = req.body;
|
|
let categories = databaseConnection.all("SELECT DISTINCT category FROM categories;", function(err,rows){res.send(200, rows)});
|
|
//res.send(200, categories);
|
|
console.log("Received category")
|
|
return next();
|
|
});
|
|
|
|
/* Main attraction: insert new categories and other data into the DB. Change name to addCategoriesToDatabase*/
|
|
function addToDatabase(input, uid){
|
|
let inputType = null;
|
|
let uuid = uid.split("=");
|
|
uuid = uuid[1];
|
|
let sqlUUID = "SELECT uuid FROM users WHERE uuid=\'" + uuid + "\'";
|
|
|
|
/* Verify message intergrity. */
|
|
let verification_hashsum = calculateSHA256(input);
|
|
console.log("UUID: ", sqlUUID, " Message verification hash sum: ", verification_hashsum);
|
|
|
|
/* Prepare insertion. */
|
|
databaseConnection.serialize(function(){
|
|
databaseConnection.prepare(sqlUUID);
|
|
});
|
|
|
|
for(var i in input){
|
|
inputType = i;
|
|
try{
|
|
let com = JSON.parse(inputType)
|
|
console.log("Command: " , inputType, com.username, com.fullname);
|
|
}
|
|
catch{
|
|
console.log("This is not a username.")
|
|
}
|
|
}
|
|
|
|
if(JSON.parse(inputType).username){
|
|
console.log("NOW WE TALK");
|
|
|
|
/* Insert username and UUID into DB.
|
|
databaseConnection.serialize(function(){
|
|
let query = databaseConnection.prepare("INSERT OR IGNORE INTO users (username) VALUES(?,?)");
|
|
query.run("Andersson", uuid);
|
|
query.finalize();
|
|
});*/
|
|
}
|
|
|
|
if(inputType == "webpage"){
|
|
/* Open up a connection. */
|
|
databaseConnection.serialize(function(){
|
|
/* Prepare queries for each table. */
|
|
let webpageQuery = databaseConnection.prepare("REPLACE INTO webpage (url, domain, sha256, md5, timestamp, uuid) VALUES(?,?,?,?,?,?)");
|
|
let categoryQuery = databaseConnection.prepare("REPLACE INTO categories (category, sha256, uuid) VALUES(?,?,?)");
|
|
let noteQuery = databaseConnection.prepare("REPLACE INTO notes (note, sha256, uuid) VALUES(?,?,?)");
|
|
let highlightedTextQuery = databaseConnection.prepare("REPLACE INTO highlightedText(highlightedText, sha256, uuid) VALUES(?,?,?)");
|
|
|
|
/* Extract key/values from receivd JSON objects.*/
|
|
let firstKey = Object.keys(input);
|
|
let firstValue = Object.values(input);
|
|
let secondKey = Object.keys(firstValue[0]);
|
|
let secondValue = Object.values(firstValue[0]);
|
|
let webpage = secondValue[0];
|
|
let url = webpage.url;
|
|
webpageQuery.run(
|
|
webpage.url,
|
|
webpage.domain,
|
|
webpage.sha256,
|
|
webpage.md5,
|
|
webpage.timestamp,
|
|
uuid,
|
|
);
|
|
|
|
/* Call D3-Collector and time*/
|
|
let date = new Date();
|
|
let t1 = performance();
|
|
let t2 = null;
|
|
let h = date.getHours();
|
|
let m = "0" + date.getMinutes();
|
|
let s = "0" + date.getSeconds();
|
|
let ss = "0" + date.getMilliseconds();
|
|
let formattedTime = h + ':' + m.substr(-2) + ':' + s.substr(-2) + '.' + ss.substr(-2);
|
|
|
|
let ok = archive_webpage(webpage.url);
|
|
|
|
if(ok != null){
|
|
t2 = performance();
|
|
}
|
|
|
|
let sync_time = t2-t1;
|
|
console.log("Fetched .onion page in: ", sync_time, " At: ", formattedTime)
|
|
log(("Fetched .onion page " + webpage.url + " in: " + sync_time + " At: " + formattedTime+"\n\n"))
|
|
|
|
/* Reset */
|
|
t1 = null;
|
|
t1 = null;
|
|
sync_time = null;
|
|
|
|
/* Parse arrays and refer to the correct SHA256. */
|
|
if(webpage.category != "N/A"){
|
|
for(var cat in webpage.category){
|
|
categoryQuery.run(webpage.category[cat], webpage.sha256, uuid);
|
|
}
|
|
}
|
|
for(var note in webpage.notes){
|
|
console.log("|- looping through notes: ", webpage.notes[note]);
|
|
noteQuery.run(webpage.notes[note], webpage.sha256, uuid);
|
|
}
|
|
for(var hlt in webpage.text){
|
|
highlightedTextQuery.run(webpage.text[hlt], webpage.sha256, uuid);
|
|
}
|
|
|
|
/* Write to database. */
|
|
|
|
t1 = performance();
|
|
webpageQuery.finalize();
|
|
t2 = performance();
|
|
sync_time = t2 - t1;
|
|
//t1 = performance();
|
|
//webpageQuery.finalize();
|
|
//2 = performance();
|
|
//sync_time = t2 - t1;
|
|
console.log("DB web page insert time: ", sync_time);
|
|
t1 = performance();
|
|
categoryQuery.finalize();
|
|
t2 = performance();
|
|
sync_time = t2-t1;
|
|
console.log("DB category insert time: ", sync_time);
|
|
t1 = performance();
|
|
noteQuery.finalize();
|
|
t2 = performance();
|
|
sync_time = t2-t1;
|
|
console.log("DB note insert time: ", sync_time);
|
|
t1 = performance();
|
|
highlightedTextQuery.finalize();
|
|
t2 = performance();
|
|
sync_time = t2-t1;
|
|
console.log("DB highlightedText page insert time: ", sync_time);
|
|
});
|
|
}
|
|
if(inputType == "category"){
|
|
databaseConnection.serialize(function(){
|
|
let query = databaseConnection.prepare("INSERT INTO categories (category,sha256, uuid) VALUES(?,?,?)");
|
|
let stKey = Object.keys(input);
|
|
let stValue = Object.values(input);
|
|
let ndValue = Object.values(stValue[0]);
|
|
console.log("Only categories being added: ", ndValue[0].category);
|
|
let categories = ndValue[0].category;
|
|
let sha256 = ndValue[0].sha256;
|
|
|
|
if(categories != "N/A"){
|
|
for(var y in categories){
|
|
console.log("|-- ", categories[y]);
|
|
query.run(categories[y], sha256, uuid);
|
|
}
|
|
}
|
|
query.finalize();
|
|
});
|
|
}
|
|
|
|
return verification_hashsum;
|
|
verification_hashsum = null;
|
|
}
|
|
|
|
/* Invoke D3-Collector to archive the web page the URLs are pointing to. */
|
|
function archive_webpage(url){
|
|
let success = false;
|
|
const spawn = require("child_process").spawn;
|
|
const spawnedShell = spawn('sh',["../D3-Collector/start-d3-collector.sh"]);
|
|
console.log("Spawning .sh to get .onion sites.", url);
|
|
// Success or not?
|
|
spawnedShell.stdout.on('data', (data) => {
|
|
console.log("Data back from D3-Collector: ", data)
|
|
if(data != null){
|
|
success = true;
|
|
}
|
|
});
|
|
|
|
return success;
|
|
}
|
|
|
|
/* Close connection */
|
|
function closeDatabaseConnection(){
|
|
databaseConnection.close((err) =>{
|
|
if(err){
|
|
console.error(err.message);
|
|
}
|
|
});
|
|
}
|
|
/* Calculate SHA256 of any input. */
|
|
function calculateSHA256(json_candidate){
|
|
const hash = crypto.createHash('sha256');
|
|
let str = JSON.stringify(json_candidate);
|
|
hash.update(str);
|
|
|
|
return hash.copy().digest('hex');
|
|
hash.update("");
|
|
str = "";
|
|
console.log("Calculating hash sums...")
|
|
}
|
|
|
|
function log(log_item){
|
|
fs.writeFile('d3-centraliser.log', log_item, function (err) {
|
|
if (err) return console.log(err);
|
|
});
|
|
}
|
|
|
|
/* Start up the server */
|
|
server.listen(8080, "10.1.100.17" , function (){
|
|
console.log('%s listening at %s', server.name, server.url);
|
|
console.log("And other sufttt.");
|
|
});
|