feat: introduce intersection calculation of charts

This commit is contained in:
syuilo 2022-02-09 03:46:58 +09:00
parent eb894c330f
commit 7fcd9435f3
15 changed files with 188 additions and 18 deletions

View file

@ -0,0 +1,47 @@
const { MigrationInterface, QueryRunner } = require("typeorm");
module.exports = class chartV141644344266289 {
name = 'chartV141644344266289'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___users"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___users"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___notedUsers"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___notedUsers"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___users"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___users"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___notedUsers"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___notedUsers"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___readWrite" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___read" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___read" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___write" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___write" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___readWrite" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___read" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___read" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___write" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___write" smallint NOT NULL DEFAULT '0'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___write"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___write"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___read"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___read"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___readWrite"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___write"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___write"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___read"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___read"`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___readWrite"`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___notedUsers" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___notedUsers" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___users" integer NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___users" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___notedUsers" smallint NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___notedUsers" character varying array NOT NULL DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___users" integer NOT NULL DEFAULT '0'`);
await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___users" character varying array NOT NULL DEFAULT '{}'`);
}
}

View file

@ -80,7 +80,7 @@ export default define(meta, async (ps, user) => {
const timeline = await query.take(ps.limit!).getMany(); const timeline = await query.take(ps.limit!).getMany();
if (user) activeUsersChart.update(user); if (user) activeUsersChart.read(user);
return await Notes.packMany(timeline, user); return await Notes.packMany(timeline, user);
}); });

View file

@ -96,7 +96,7 @@ export default define(meta, async (ps, user) => {
process.nextTick(() => { process.nextTick(() => {
if (user) { if (user) {
activeUsersChart.update(user); activeUsersChart.read(user);
} }
}); });

View file

@ -153,7 +153,7 @@ export default define(meta, async (ps, user) => {
process.nextTick(() => { process.nextTick(() => {
if (user) { if (user) {
activeUsersChart.update(user); activeUsersChart.read(user);
} }
}); });

View file

@ -122,7 +122,7 @@ export default define(meta, async (ps, user) => {
process.nextTick(() => { process.nextTick(() => {
if (user) { if (user) {
activeUsersChart.update(user); activeUsersChart.read(user);
} }
}); });

View file

@ -145,7 +145,7 @@ export default define(meta, async (ps, user) => {
process.nextTick(() => { process.nextTick(() => {
if (user) { if (user) {
activeUsersChart.update(user); activeUsersChart.read(user);
} }
}); });

View file

@ -142,7 +142,7 @@ export default define(meta, async (ps, user) => {
const timeline = await query.take(ps.limit!).getMany(); const timeline = await query.take(ps.limit!).getMany();
activeUsersChart.update(user); activeUsersChart.read(user);
return await Notes.packMany(timeline, user); return await Notes.packMany(timeline, user);
}); });

View file

@ -23,9 +23,9 @@ export default class ActiveUsersChart extends Chart<typeof schema> {
} }
@autobind @autobind
public async update(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> { public async read(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
await this.commit({ await this.commit({
'users': [user.id], 'read': [user.id],
'registeredWithinWeek': (Date.now() - user.createdAt.getTime() < week) ? [user.id] : [], 'registeredWithinWeek': (Date.now() - user.createdAt.getTime() < week) ? [user.id] : [],
'registeredWithinMonth': (Date.now() - user.createdAt.getTime() < month) ? [user.id] : [], 'registeredWithinMonth': (Date.now() - user.createdAt.getTime() < month) ? [user.id] : [],
'registeredWithinYear': (Date.now() - user.createdAt.getTime() < year) ? [user.id] : [], 'registeredWithinYear': (Date.now() - user.createdAt.getTime() < year) ? [user.id] : [],
@ -36,9 +36,9 @@ export default class ActiveUsersChart extends Chart<typeof schema> {
} }
@autobind @autobind
public async noted(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> { public async write(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
await this.commit({ await this.commit({
'notedUsers': [user.id], 'write': [user.id],
}); });
} }
} }

View file

@ -3,8 +3,9 @@ import Chart from '../../core';
export const name = 'activeUsers'; export const name = 'activeUsers';
export const schema = { export const schema = {
'users': { uniqueIncrement: true }, 'readWrite': { intersection: ['read', 'write'], range: 'small' },
'notedUsers': { uniqueIncrement: true, range: 'small' }, 'read': { uniqueIncrement: true, range: 'small' },
'write': { uniqueIncrement: true, range: 'small' },
'registeredWithinWeek': { uniqueIncrement: true, range: 'small' }, 'registeredWithinWeek': { uniqueIncrement: true, range: 'small' },
'registeredWithinMonth': { uniqueIncrement: true, range: 'small' }, 'registeredWithinMonth': { uniqueIncrement: true, range: 'small' },
'registeredWithinYear': { uniqueIncrement: true, range: 'small' }, 'registeredWithinYear': { uniqueIncrement: true, range: 'small' },

View file

@ -0,0 +1,11 @@
import Chart from '../../core';
export const name = 'testIntersection';
export const schema = {
'a': { uniqueIncrement: true },
'b': { uniqueIncrement: true },
'aAndB': { intersection: ['a', 'b'] },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -0,0 +1,32 @@
import autobind from 'autobind-decorator';
import Chart, { KVs } from '../core';
import { name, schema } from './entities/test-intersection';
/**
* For testing
*/
// eslint-disable-next-line import/no-default-export
export default class TestIntersectionChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
@autobind
protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
@autobind
public async addA(key: string): Promise<void> {
await this.commit({
a: [key],
});
}
@autobind
public async addB(key: string): Promise<void> {
await this.commit({
b: [key],
});
}
}

View file

@ -46,6 +46,8 @@ const removeDuplicates = (array: any[]) => Array.from(new Set(array));
type Schema = Record<string, { type Schema = Record<string, {
uniqueIncrement?: boolean; uniqueIncrement?: boolean;
intersection?: string[] | ReadonlyArray<string>;
range?: 'big' | 'small' | 'medium'; range?: 'big' | 'small' | 'medium';
// previousな値を引き継ぐかどうか // previousな値を引き継ぐかどうか
@ -384,6 +386,33 @@ export default abstract class Chart<T extends Schema> {
} }
} }
// compute intersection
// TODO: intersectionに指定されたカラムがintersectionだった場合の対応
for (const [k, v] of Object.entries(this.schema)) {
const intersection = v.intersection;
if (intersection) {
const name = columnPrefix + k.replaceAll('.', columnDot);
const firstKey = intersection[0];
const firstTempColumnName = uniqueTempColumnPrefix + firstKey.replaceAll('.', columnDot);
const currentValuesForHour = new Set([...(finalDiffs[firstKey] ?? []), ...logHour[firstTempColumnName]]);
const currentValuesForDay = new Set([...(finalDiffs[firstKey] ?? []), ...logDay[firstTempColumnName]]);
for (let i = 1; i < intersection.length; i++) {
const targetKey = intersection[i];
const targetTempColumnName = uniqueTempColumnPrefix + targetKey.replaceAll('.', columnDot);
const targetValuesForHour = new Set([...(finalDiffs[targetKey] ?? []), ...logHour[targetTempColumnName]]);
const targetValuesForDay = new Set([...(finalDiffs[targetKey] ?? []), ...logDay[targetTempColumnName]]);
currentValuesForHour.forEach(v => {
if (!targetValuesForHour.has(v)) currentValuesForHour.delete(v);
});
currentValuesForDay.forEach(v => {
if (!targetValuesForDay.has(v)) currentValuesForDay.delete(v);
});
}
queryForHour[name] = currentValuesForHour.size;
queryForDay[name] = currentValuesForDay.size;
}
}
// ログ更新 // ログ更新
await Promise.all([ await Promise.all([
this.repositoryForHour.createQueryBuilder() this.repositoryForHour.createQueryBuilder()

View file

@ -297,7 +297,7 @@ export default async (user: { id: User['id']; username: User['username']; host:
} }
if (!silent) { if (!silent) {
if (Users.isLocalUser(user)) activeUsersChart.noted(user); if (Users.isLocalUser(user)) activeUsersChart.write(user);
// 未読通知を作成 // 未読通知を作成
if (data.visibility === 'specified') { if (data.visibility === 'specified') {

View file

@ -6,14 +6,17 @@ import { async, initTestDb } from './utils';
import TestChart from '../src/services/chart/charts/test'; import TestChart from '../src/services/chart/charts/test';
import TestGroupedChart from '../src/services/chart/charts/test-grouped'; import TestGroupedChart from '../src/services/chart/charts/test-grouped';
import TestUniqueChart from '../src/services/chart/charts/test-unique'; import TestUniqueChart from '../src/services/chart/charts/test-unique';
import TestIntersectionChart from '../src/services/chart/charts/test-intersection';
import * as _TestChart from '../src/services/chart/charts/entities/test'; import * as _TestChart from '../src/services/chart/charts/entities/test';
import * as _TestGroupedChart from '../src/services/chart/charts/entities/test-grouped'; import * as _TestGroupedChart from '../src/services/chart/charts/entities/test-grouped';
import * as _TestUniqueChart from '../src/services/chart/charts/entities/test-unique'; import * as _TestUniqueChart from '../src/services/chart/charts/entities/test-unique';
import * as _TestIntersectionChart from '../src/services/chart/charts/entities/test-intersection';
describe('Chart', () => { describe('Chart', () => {
let testChart: TestChart; let testChart: TestChart;
let testGroupedChart: TestGroupedChart; let testGroupedChart: TestGroupedChart;
let testUniqueChart: TestUniqueChart; let testUniqueChart: TestUniqueChart;
let testIntersectionChart: TestIntersectionChart;
let clock: lolex.Clock; let clock: lolex.Clock;
beforeEach(async(async () => { beforeEach(async(async () => {
@ -21,11 +24,13 @@ describe('Chart', () => {
_TestChart.entity.hour, _TestChart.entity.day, _TestChart.entity.hour, _TestChart.entity.day,
_TestGroupedChart.entity.hour, _TestGroupedChart.entity.day, _TestGroupedChart.entity.hour, _TestGroupedChart.entity.day,
_TestUniqueChart.entity.hour, _TestUniqueChart.entity.day, _TestUniqueChart.entity.hour, _TestUniqueChart.entity.day,
_TestIntersectionChart.entity.hour, _TestIntersectionChart.entity.day,
]); ]);
testChart = new TestChart(); testChart = new TestChart();
testGroupedChart = new TestGroupedChart(); testGroupedChart = new TestGroupedChart();
testUniqueChart = new TestUniqueChart(); testUniqueChart = new TestUniqueChart();
testIntersectionChart = new TestIntersectionChart();
clock = lolex.install({ clock = lolex.install({
now: new Date(Date.UTC(2000, 0, 1, 0, 0, 0)) now: new Date(Date.UTC(2000, 0, 1, 0, 0, 0))
@ -426,6 +431,45 @@ describe('Chart', () => {
foo: [2, 0, 0], foo: [2, 0, 0],
}); });
})); }));
describe('Intersection', () => {
it('条件が満たされていない場合はカウントされない', async(async () => {
await testIntersectionChart.addA('alice');
await testIntersectionChart.addA('bob');
await testIntersectionChart.addB('carol');
await testIntersectionChart.save();
const chartHours = await testUniqueChart.getChart('hour', 3, null);
const chartDays = await testUniqueChart.getChart('day', 3, null);
assert.deepStrictEqual(chartHours, {
aAndB: [0, 0, 0],
});
assert.deepStrictEqual(chartDays, {
aAndB: [0, 0, 0],
});
}));
it('条件が満たされている場合にカウントされる', async(async () => {
await testIntersectionChart.addA('alice');
await testIntersectionChart.addA('bob');
await testIntersectionChart.addB('carol');
await testIntersectionChart.addB('alice');
await testIntersectionChart.save();
const chartHours = await testUniqueChart.getChart('hour', 3, null);
const chartDays = await testUniqueChart.getChart('day', 3, null);
assert.deepStrictEqual(chartHours, {
aAndB: [1, 0, 0],
});
assert.deepStrictEqual(chartDays, {
aAndB: [1, 0, 0],
});
}));
});
}); });
describe('Resync', () => { describe('Resync', () => {

View file

@ -69,6 +69,7 @@ const colors = {
yellow: '#FEB019', yellow: '#FEB019',
red: '#FF4560', red: '#FF4560',
purple: '#e300db', purple: '#e300db',
orange: '#fe6919',
}; };
const colorSets = [colors.blue, colors.green, colors.yellow, colors.red, colors.purple]; const colorSets = [colors.blue, colors.green, colors.yellow, colors.red, colors.purple];
const getColor = (i) => { const getColor = (i) => {
@ -518,15 +519,20 @@ export default defineComponent({
const raw = await os.api('charts/active-users', { limit: props.limit, span: props.span }); const raw = await os.api('charts/active-users', { limit: props.limit, span: props.span });
return { return {
series: [{ series: [{
name: 'Active', name: 'Read & Write',
type: 'area', type: 'area',
data: format(raw.users), data: format(raw.readWrite),
color: '#888888', color: colors.orange,
}, { }, {
name: 'Noted', name: 'Write',
type: 'area', type: 'area',
data: format(raw.notedUsers), data: format(raw.write),
color: colors.blue, color: colors.blue,
}, {
name: 'Read',
type: 'area',
data: format(raw.read),
color: '#888888',
}, { }, {
name: '< Week', name: '< Week',
type: 'area', type: 'area',