🧹 cleanup postgres + mysql

pull/680/head
Philipp Dormann 2021-05-22 18:09:28 +02:00
parent 0e498a5701
commit e667928863
16 changed files with 667 additions and 752 deletions

1
.gitignore vendored
View File

@ -11,7 +11,6 @@
# next.js
/.next/
/out/
/prisma/schema.prisma
# production
/build

View File

@ -1,63 +1,30 @@
# umami
# umami-sqlite
Modded version of [Umami Analytics](https://umami.is) backed by a local SQLite database
Umami is a simple, fast, website analytics alternative to Google Analytics.
## Getting started
- Node.js >= 14.17.0
A detailed getting started guide can be found at [https://umami.is/docs/](https://umami.is/docs/)
## Installing from source
### Requirements
- A server with Node.js 10.13 or newer
- A database (MySQL or Postgresql)
### Get the source code and install packages
### Install Dependencies
```
git clone https://github.com/mikecao/umami.git
cd umami
npm install
yarn
```
### Create database tables
Umami supports [MySQL](https://www.mysql.com/) and [Postgresql](https://www.postgresql.org/).
Create a database for your Umami installation and install the tables with the included scripts.
For MySQL:
### Run db migration
```
mysql -u username -p databasename < sql/schema.mysql.sql
yarn prisma migrate dev
```
For Postgresql:
```
psql -h hostname -U username -d databasename -f sql/schema.postgresql.sql
```
This will also create a login account with username **admin** and password **umami**.
### Configure umami
Create an `.env` file with the following
```
DATABASE_URL=(connection url)
HASH_SALT=(any random string)
```
The connection url is in the following format:
```
postgresql://username:mypassword@localhost:5432/mydb
mysql://username:mypassword@localhost:3306/mydb
```
The `HASH_SALT` is used to generate unique values for your installation.
### Build the application
```bash
@ -72,36 +39,4 @@ npm start
By default this will launch the application on `http://localhost:3000`. You will need to either
[proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) requests from your web server
or change the [port](https://nextjs.org/docs/api-reference/cli#production) to serve the application directly.
## Installing with Docker
To build the umami container and start up a Postgres database, run:
```bash
docker-compose up
```
Alternatively, to pull just the Umami Docker image with PostgreSQL support:
```bash
docker pull ghcr.io/mikecao/umami:postgresql-latest
```
Or with MySQL support:
```bash
docker pull ghcr.io/mikecao/umami:mysql-latest
```
## Getting updates
To get the latest features, simply do a pull, install any new dependencies, and rebuild:
```bash
git pull
npm install
npm run build
```
## License
MIT
or change the [port](https://nextjs.org/docs/api-reference/cli#production) to serve the application directly.

View File

@ -6,16 +6,5 @@ services:
- 50001:3000
environment:
HASH_SALT: xSSHW4A2w0e3vxLUp46XL9OzCGKvxEnA
# depends_on:
# - db
# db:
# image: postgres:13.3-alpine
# environment:
# POSTGRES_DB: umami
# POSTGRES_USER: umami
# POSTGRES_PASSWORD: umami
# volumes:
# - ./sql/schema.postgresql.sql:/docker-entrypoint-initdb.d/schema.postgresql.sql:ro
# - umami-db-data:/var/lib/postgresql/data
# volumes:
# umami-db-data:
volumes:
- ./db:/app/prisma/db

View File

@ -20,10 +20,7 @@
"build-geo": "node scripts/build-geo.js",
"build-db-schema": "dotenv prisma introspect",
"build-db-client": "dotenv prisma generate",
"download-country-names": "node scripts/download-country-names.js",
"loadtest": "node scripts/loadtest.js",
"loadtest:medium": "node scripts/loadtest.js --weight=medium",
"loadtest:heavy": "node scripts/loadtest.js --weight=heavy --verbose"
"download-country-names": "node scripts/download-country-names.js"
},
"dependencies": {
"@prisma/client": "2.21.2",
@ -77,8 +74,8 @@
"del": "6.0.0",
"dotenv-cli": "4.0.0",
"extract-react-intl-messages": "4.1.1",
"loadtest": "5.1.2",
"npm-run-all": "4.1.5",
"pkg": "^5.2.0",
"postcss-flexbugs-fixes": "5.0.2",
"postcss-import": "13.0.0",
"postcss-preset-env": "6.7.0",

View File

@ -1,99 +0,0 @@
-- CreateTable
CREATE TABLE `account` (
`user_id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`username` VARCHAR(255) NOT NULL,
`password` VARCHAR(60) NOT NULL,
`is_admin` BOOLEAN NOT NULL DEFAULT false,
`created_at` TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP(0),
`updated_at` TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP(0),
UNIQUE INDEX `account.username_unique`(`username`),
PRIMARY KEY (`user_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `event` (
`event_id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`website_id` INTEGER UNSIGNED NOT NULL,
`session_id` INTEGER UNSIGNED NOT NULL,
`created_at` TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP(0),
`url` VARCHAR(500) NOT NULL,
`event_type` VARCHAR(50) NOT NULL,
`event_value` VARCHAR(50) NOT NULL,
INDEX `event_created_at_idx`(`created_at`),
INDEX `event_session_id_idx`(`session_id`),
INDEX `event_website_id_idx`(`website_id`),
PRIMARY KEY (`event_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `pageview` (
`view_id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`website_id` INTEGER UNSIGNED NOT NULL,
`session_id` INTEGER UNSIGNED NOT NULL,
`created_at` TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP(0),
`url` VARCHAR(500) NOT NULL,
`referrer` VARCHAR(500),
INDEX `pageview_created_at_idx`(`created_at`),
INDEX `pageview_session_id_idx`(`session_id`),
INDEX `pageview_website_id_created_at_idx`(`website_id`, `created_at`),
INDEX `pageview_website_id_idx`(`website_id`),
INDEX `pageview_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`),
PRIMARY KEY (`view_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `session` (
`session_id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`session_uuid` VARCHAR(36) NOT NULL,
`website_id` INTEGER UNSIGNED NOT NULL,
`created_at` TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP(0),
`hostname` VARCHAR(100),
`browser` VARCHAR(20),
`os` VARCHAR(20),
`device` VARCHAR(20),
`screen` VARCHAR(11),
`language` VARCHAR(35),
`country` CHAR(2),
UNIQUE INDEX `session.session_uuid_unique`(`session_uuid`),
INDEX `session_created_at_idx`(`created_at`),
INDEX `session_website_id_idx`(`website_id`),
PRIMARY KEY (`session_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `website` (
`website_id` INTEGER UNSIGNED NOT NULL AUTO_INCREMENT,
`website_uuid` VARCHAR(36) NOT NULL,
`user_id` INTEGER UNSIGNED NOT NULL,
`name` VARCHAR(100) NOT NULL,
`domain` VARCHAR(500),
`share_id` VARCHAR(64),
`created_at` TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP(0),
UNIQUE INDEX `website.website_uuid_unique`(`website_uuid`),
UNIQUE INDEX `website.share_id_unique`(`share_id`),
INDEX `website_user_id_idx`(`user_id`),
PRIMARY KEY (`website_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- AddForeignKey
ALTER TABLE `event` ADD FOREIGN KEY (`session_id`) REFERENCES `session`(`session_id`) ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `event` ADD FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `pageview` ADD FOREIGN KEY (`session_id`) REFERENCES `session`(`session_id`) ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `pageview` ADD FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `session` ADD FOREIGN KEY (`website_id`) REFERENCES `website`(`website_id`) ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE `website` ADD FOREIGN KEY (`user_id`) REFERENCES `account`(`user_id`) ON DELETE CASCADE ON UPDATE CASCADE;

View File

@ -1,3 +0,0 @@
# Please do not edit this file manually
# It should be added in your version-control system (i.e. Git)
provider = "mysql"

View File

@ -1 +0,0 @@
../schema.mysql.prisma

View File

@ -1 +0,0 @@
../seed.js

View File

@ -1,129 +0,0 @@
-- CreateTable
CREATE TABLE "account" (
"user_id" SERIAL NOT NULL,
"username" VARCHAR(255) NOT NULL,
"password" VARCHAR(60) NOT NULL,
"is_admin" BOOLEAN NOT NULL DEFAULT false,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY ("user_id")
);
-- CreateTable
CREATE TABLE "event" (
"event_id" SERIAL NOT NULL,
"website_id" INTEGER NOT NULL,
"session_id" INTEGER NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
"url" VARCHAR(500) NOT NULL,
"event_type" VARCHAR(50) NOT NULL,
"event_value" VARCHAR(50) NOT NULL,
PRIMARY KEY ("event_id")
);
-- CreateTable
CREATE TABLE "pageview" (
"view_id" SERIAL NOT NULL,
"website_id" INTEGER NOT NULL,
"session_id" INTEGER NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
"url" VARCHAR(500) NOT NULL,
"referrer" VARCHAR(500),
PRIMARY KEY ("view_id")
);
-- CreateTable
CREATE TABLE "session" (
"session_id" SERIAL NOT NULL,
"session_uuid" UUID NOT NULL,
"website_id" INTEGER NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
"hostname" VARCHAR(100),
"browser" VARCHAR(20),
"os" VARCHAR(20),
"device" VARCHAR(20),
"screen" VARCHAR(11),
"language" VARCHAR(35),
"country" CHAR(2),
PRIMARY KEY ("session_id")
);
-- CreateTable
CREATE TABLE "website" (
"website_id" SERIAL NOT NULL,
"website_uuid" UUID NOT NULL,
"user_id" INTEGER NOT NULL,
"name" VARCHAR(100) NOT NULL,
"domain" VARCHAR(500),
"share_id" VARCHAR(64),
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY ("website_id")
);
-- CreateIndex
CREATE UNIQUE INDEX "account.username_unique" ON "account"("username");
-- CreateIndex
CREATE INDEX "event_created_at_idx" ON "event"("created_at");
-- CreateIndex
CREATE INDEX "event_session_id_idx" ON "event"("session_id");
-- CreateIndex
CREATE INDEX "event_website_id_idx" ON "event"("website_id");
-- CreateIndex
CREATE INDEX "pageview_created_at_idx" ON "pageview"("created_at");
-- CreateIndex
CREATE INDEX "pageview_session_id_idx" ON "pageview"("session_id");
-- CreateIndex
CREATE INDEX "pageview_website_id_created_at_idx" ON "pageview"("website_id", "created_at");
-- CreateIndex
CREATE INDEX "pageview_website_id_idx" ON "pageview"("website_id");
-- CreateIndex
CREATE INDEX "pageview_website_id_session_id_created_at_idx" ON "pageview"("website_id", "session_id", "created_at");
-- CreateIndex
CREATE UNIQUE INDEX "session.session_uuid_unique" ON "session"("session_uuid");
-- CreateIndex
CREATE INDEX "session_created_at_idx" ON "session"("created_at");
-- CreateIndex
CREATE INDEX "session_website_id_idx" ON "session"("website_id");
-- CreateIndex
CREATE UNIQUE INDEX "website.website_uuid_unique" ON "website"("website_uuid");
-- CreateIndex
CREATE UNIQUE INDEX "website.share_id_unique" ON "website"("share_id");
-- CreateIndex
CREATE INDEX "website_user_id_idx" ON "website"("user_id");
-- AddForeignKey
ALTER TABLE "event" ADD FOREIGN KEY ("session_id") REFERENCES "session"("session_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "event" ADD FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "pageview" ADD FOREIGN KEY ("session_id") REFERENCES "session"("session_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "pageview" ADD FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "session" ADD FOREIGN KEY ("website_id") REFERENCES "website"("website_id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "website" ADD FOREIGN KEY ("user_id") REFERENCES "account"("user_id") ON DELETE CASCADE ON UPDATE CASCADE;

View File

@ -1,3 +0,0 @@
# Please do not edit this file manually
# It should be added in your version-control system (i.e. Git)
provider = "postgresql"

View File

@ -1 +0,0 @@
../schema.postgresql.prisma

View File

@ -1 +0,0 @@
../seed.js

View File

@ -1,87 +0,0 @@
generator client {
provider = "prisma-client-js"
}
datasource db {
provider = "postgresql"
url = env("DATABASE_URL")
}
model account {
user_id Int @id @default(autoincrement())
username String @unique @db.VarChar(255)
password String @db.VarChar(60)
is_admin Boolean @default(false)
created_at DateTime? @default(now()) @db.Timestamptz(6)
updated_at DateTime? @default(now()) @db.Timestamptz(6)
website website[]
}
model event {
event_id Int @id @default(autoincrement())
website_id Int
session_id Int
created_at DateTime? @default(now()) @db.Timestamptz(6)
url String @db.VarChar(500)
event_type String @db.VarChar(50)
event_value String @db.VarChar(50)
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
@@index([created_at], name: "event_created_at_idx")
@@index([session_id], name: "event_session_id_idx")
@@index([website_id], name: "event_website_id_idx")
}
model pageview {
view_id Int @id @default(autoincrement())
website_id Int
session_id Int
created_at DateTime? @default(now()) @db.Timestamptz(6)
url String @db.VarChar(500)
referrer String? @db.VarChar(500)
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
@@index([created_at], name: "pageview_created_at_idx")
@@index([session_id], name: "pageview_session_id_idx")
@@index([website_id, created_at], name: "pageview_website_id_created_at_idx")
@@index([website_id], name: "pageview_website_id_idx")
@@index([website_id, session_id, created_at], name: "pageview_website_id_session_id_created_at_idx")
}
model session {
session_id Int @id @default(autoincrement())
session_uuid String @unique @db.Uuid
website_id Int
created_at DateTime? @default(now()) @db.Timestamptz(6)
hostname String? @db.VarChar(100)
browser String? @db.VarChar(20)
os String? @db.VarChar(20)
device String? @db.VarChar(20)
screen String? @db.VarChar(11)
language String? @db.VarChar(35)
country String? @db.Char(2)
website website @relation(fields: [website_id], references: [website_id])
event event[]
pageview pageview[]
@@index([created_at], name: "session_created_at_idx")
@@index([website_id], name: "session_website_id_idx")
}
model website {
website_id Int @id @default(autoincrement())
website_uuid String @unique @db.Uuid
user_id Int
name String @db.VarChar(100)
domain String? @db.VarChar(500)
share_id String? @unique @db.VarChar(64)
created_at DateTime? @default(now()) @db.Timestamptz(6)
account account @relation(fields: [user_id], references: [user_id])
event event[]
pageview pageview[]
session session[]
@@index([user_id], name: "website_user_id_idx")
}

87
prisma/schema.prisma Normal file
View File

@ -0,0 +1,87 @@
generator client {
provider = "prisma-client-js"
}
datasource db {
provider = "sqlite"
url = "file:./db/umami.db"
}
model account {
user_id Int @id @default(autoincrement())
username String @unique
password String
is_admin Boolean @default(false)
created_at DateTime? @default(now())
updated_at DateTime? @default(now())
website website[]
}
model event {
event_id Int @id @default(autoincrement())
website_id Int
session_id Int
created_at DateTime? @default(now())
url String
event_type String
event_value String
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
@@index([created_at], name: "event_created_at_idx")
@@index([session_id], name: "event_session_id_idx")
@@index([website_id], name: "event_website_id_idx")
}
model pageview {
view_id Int @id @default(autoincrement())
website_id Int
session_id Int
created_at DateTime? @default(now())
url String
referrer String?
session session @relation(fields: [session_id], references: [session_id])
website website @relation(fields: [website_id], references: [website_id])
@@index([created_at], name: "pageview_created_at_idx")
@@index([session_id], name: "pageview_session_id_idx")
@@index([website_id, created_at], name: "pageview_website_id_created_at_idx")
@@index([website_id], name: "pageview_website_id_idx")
@@index([website_id, session_id, created_at], name: "pageview_website_id_session_id_created_at_idx")
}
model session {
session_id Int @id @default(autoincrement())
session_uuid String @unique
website_id Int
created_at DateTime? @default(now())
hostname String?
browser String?
os String?
device String?
screen String?
language String?
country String?
website website @relation(fields: [website_id], references: [website_id])
event event[]
pageview pageview[]
@@index([created_at], name: "session_created_at_idx")
@@index([website_id], name: "session_website_id_idx")
}
model website {
website_id Int @id @default(autoincrement())
website_uuid String @unique
user_id Int
name String
domain String?
share_id String? @unique
created_at DateTime? @default(now())
account account @relation(fields: [user_id], references: [user_id])
event event[]
pageview pageview[]
session session[]
@@index([user_id], name: "website_user_id_idx")
}

View File

@ -1,142 +0,0 @@
const loadtest = require('loadtest');
const chalk = require('chalk');
const trunc = num => +num.toFixed(1);
/**
* Example invocations:
*
* npm run loadtest -- --weight=heavy
* npm run loadtest -- --weight=heavy --verbose
* npm run loadtest -- --weight=single --verbose
* npm run loadtest -- --weight=medium
*/
/**
* Command line arguments like --weight=heavy and --verbose use this object
* If you are providing _alternative_ configs, use --weight
* e.g. add --weight=ultra then add commandlineOptions.ultra={}
* --verbose can be combied with any weight.
*/
const commandlineOptions = {
single: {
concurrency: 1,
requestsPerSecond: 1,
maxSeconds: 5,
maxRequests: 1,
},
// Heavy can saturate CPU which leads to requests stalling depending on machine
// Keep an eye if --verbose logs pause, or if node CPU in top is > 100.
// https://github.com/alexfernandez/loadtest#usage-donts
heavy: {
concurrency: 10,
requestsPerSecond: 200,
maxSeconds: 60,
},
// Throttled requests should not max out CPU,
medium: {
concurrency: 3,
requestsPerSecond: 5,
maxSeconds: 60,
},
verbose: { statusCallback },
};
const options = {
url: 'http://localhost:3000',
method: 'POST',
concurrency: 5,
requestsPerSecond: 5,
maxSeconds: 5,
requestGenerator: (params, options, client, callback) => {
const message = JSON.stringify(mockPageView());
options.headers['Content-Length'] = message.length;
options.headers['Content-Type'] = 'application/json';
options.body = message;
options.path = '/api/collect';
const request = client(options, callback);
request.write(message);
return request;
},
};
function getArgument() {
const weight = process.argv[2] && process.argv[2].replace('--weight=', '');
const verbose = process.argv.includes('--verbose') && 'verbose';
return [weight, verbose];
}
// Patch in all command line arguments over options object
// Must do this prior to calling `loadTest()`
getArgument().map(arg => Object.assign(options, commandlineOptions[arg]));
loadtest.loadTest(options, (error, results) => {
if (error) {
return console.error(chalk.redBright('Got an error: %s', error));
}
console.log(chalk.bold(chalk.yellow('\n--------\n')));
console.log(chalk.yellowBright('Loadtests complete:'), chalk.greenBright('success'), '\n');
prettyLogItem('Total Requests:', results.totalRequests);
prettyLogItem('Total Errors:', results.totalErrors);
prettyLogItem(
'Latency(mean/min/max)',
trunc(results.meanLatencyMs),
'/',
trunc(results.maxLatencyMs),
'/',
trunc(results.minLatencyMs),
);
if (results.totalErrors) {
console.log(chalk.redBright('*'), chalk.red('Total Errors:'), results.totalErrors);
}
if (results.errorCodes && Object.keys(results.errorCodes).length) {
console.log(chalk.redBright('*'), chalk.red('Error Codes:'), results.errorCodes);
}
});
/**
* Create a new object for each request. Note, we could randomize values here if desired.
*
* TODO: Need a better way of passing in websiteId, hostname, URL.
*
* @param {object} payload pageview payload same as sent via tracker
*/
function mockPageView(
payload = {
website: 'fcd4c7e3-ed76-439c-9121-3a0f102df126',
hostname: 'localhost',
screen: '1680x1050',
url: '/LOADTESTING',
referrer: '/REFERRER',
},
) {
return {
type: 'pageview',
payload,
};
}
// If you pass in --verbose, this function is called
function statusCallback(error, result, latency) {
if (error) {
return console.error(chalk.redBright(error));
}
console.log(
chalk.yellowBright(`\n## req #${result.requestIndex + 1} of ${latency.totalRequests}`),
);
prettyLogItem('Request elapsed milliseconds:', trunc(result.requestElapsed));
prettyLogItem(
'Latency(mean/max/min):',
trunc(latency.meanLatencyMs),
'/',
trunc(latency.maxLatencyMs),
'/',
trunc(latency.minLatencyMs),
);
}
function prettyLogItem(label, ...args) {
console.log(chalk.redBright('*'), chalk.green(label), ...args);
}

761
yarn.lock

File diff suppressed because it is too large Load Diff