From 02e52d73510fe9ffc077d87082b051d0740c67d5 Mon Sep 17 00:00:00 2001 From: pluja Date: Wed, 4 Jun 2025 19:37:33 +0000 Subject: [PATCH] Release 202506041937 --- .env.example | 46 +++++ docker-compose.yml | 25 +-- web/Dockerfile | 11 +- .../migration.sql | 2 + web/prisma/schema.prisma | 1 + web/src/actions/admin/notification.ts | 98 +++------- web/src/constants/notificationTypes.ts | 5 + web/src/lib/notifications.ts | 11 ++ web/src/lib/postgresListenerIntegration.ts | 154 +-------------- web/src/lib/sendNotifications.ts | 185 ++++++++++++++++++ web/src/pages/admin/notifications.astro | 38 +--- 11 files changed, 298 insertions(+), 278 deletions(-) create mode 100644 web/prisma/migrations/20250604164448_test_notification/migration.sql create mode 100644 web/src/lib/sendNotifications.ts diff --git a/.env.example b/.env.example index e69de29..b2c0b4a 100644 --- a/.env.example +++ b/.env.example @@ -0,0 +1,46 @@ +# Database +POSTGRES_USER=your_db_user +POSTGRES_PASSWORD=your_db_password +POSTGRES_DATABASE=your_db_name +DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database:5432/${POSTGRES_DATABASE}?schema=public" +DATABASE_UI_URL="https://db.example.com" + +# Generic Config +UPLOAD_DIR=/app/uploads +SITE_URL="https://your-site.example.com" +SOURCE_CODE_URL="https://your-source-code.example.com" +TIME_TRAP_SECRET=your_time_trap_secret +LOGS_UI_URL="https://logs.example.com" + +# Release Info +RELEASE_NUMBER= +RELEASE_DATE= + +# Redis +REDIS_URL="redis://redis:6379" + +# Crawl4AI +CRAWL4AI_BASE_URL="http://crawl4ai:11235" +CRAWL4AI_API_TOKEN=your_crawl4ai_token + +# Tor and I2P +ONION_ADDRESS="http://youronionaddress.onion" +I2P_ADDRESS="http://youri2paddress.b32.i2p" +I2P_PASS=your_i2p_password + +# Push Notifications +VAPID_PUBLIC_KEY=your_vapid_public_key +VAPID_PRIVATE_KEY=your_vapid_private_key +VAPID_SUBJECT="mailto:your-email@example.com" + +# OpenAI +OPENAI_API_KEY=your_openai_api_key +OPENAI_BASE_URL="https://your-openai-base-url.example.com" +OPENAI_MODEL=your_openai_model +OPENAI_RETRY=3 + +# Pyworker Crons +CRON_TOSREVIEW_TASK="0 0 1 * *" # Every month +CRON_USER_SENTIMENT_TASK="0 0 * * *" # Every day +CRON_COMMENT_MODERATION_TASK="0 * * * *" # Every hour +CRON_FORCE_TRIGGERS_TASK="0 2 * * *" # Every day \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index fbd8ddc..97e5242 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,8 @@ services: volumes: - database:/var/lib/postgresql/data:z restart: unless-stopped - environment: - POSTGRES_USER: ${POSTGRES_USER:-kycnot} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-kycnot} - POSTGRES_DB: ${POSTGRES_DATABASE:-kycnot} + env_file: + - .env healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-kycnot} -d ${POSTGRES_DATABASE:-kycnot}"] interval: 10s @@ -21,18 +19,15 @@ services: build: context: ./pyworker restart: always - environment: - DATABASE_URL: "postgresql://${POSTGRES_USER:-kycnot}:${POSTGRES_PASSWORD:-kycnot}@database:5432/${POSTGRES_DATABASE:-kycnot}?schema=public" - CRAWL4AI_BASE_URL: "http://crawl4ai:11235" - CRAWL4AI_API_TOKEN: ${CRAWL4AI_API_TOKEN:-testing} + env_file: + - .env crawl4ai: image: unclecode/crawl4ai:basic-amd64 expose: - "11235" - environment: - CRAWL4AI_API_TOKEN: ${CRAWL4AI_API_TOKEN:-testing} # Optional API security - MAX_CONCURRENT_TASKS: 10 + env_file: + - .env volumes: - /dev/shm:/dev/shm deploy: @@ -53,15 +48,9 @@ services: astro: build: - context: ./web + dockerfile: web/Dockerfile image: kycnotme/astro:${ASTRO_IMAGE_TAG:-latest} restart: unless-stopped - environment: - POSTGRES_USER: ${POSTGRES_USER:-kycnot} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-kycnot} - POSTGRES_DB: ${POSTGRES_DATABASE:-kycnot} - DATABASE_URL: "postgresql://${POSTGRES_USER:-kycnot}:${POSTGRES_PASSWORD:-kycnot}@database:5432/${POSTGRES_DATABASE:-kycnot}?schema=public" - REDIS_URL: "redis://redis:6379" env_file: - .env depends_on: diff --git a/web/Dockerfile b/web/Dockerfile index 9d3ec87..73f8dde 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,17 +1,20 @@ FROM node:lts AS runtime WORKDIR /app -COPY package.json package-lock.json ./ +COPY .env .env +COPY web/package.json web/package-lock.json ./ -COPY .npmrc .npmrc +COPY web/.npmrc .npmrc RUN npm ci -COPY . . +COPY web/ . ARG ASTRO_BUILD_MODE=production + # Generate Prisma client RUN npx prisma generate + # Build the application RUN npm run build -- --mode ${ASTRO_BUILD_MODE} @@ -20,7 +23,7 @@ ENV PORT=4321 EXPOSE 4321 # Add knm-migrate command script and make it executable -COPY migrate.sh /usr/local/bin/knm-migrate +COPY web/migrate.sh /usr/local/bin/knm-migrate RUN chmod +x /usr/local/bin/knm-migrate CMD ["node", "./dist/server/entry.mjs"] diff --git a/web/prisma/migrations/20250604164448_test_notification/migration.sql b/web/prisma/migrations/20250604164448_test_notification/migration.sql new file mode 100644 index 0000000..a0102a4 --- /dev/null +++ b/web/prisma/migrations/20250604164448_test_notification/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "NotificationType" ADD VALUE 'TEST'; diff --git a/web/prisma/schema.prisma b/web/prisma/schema.prisma index 0e52ad8..1aee840 100644 --- a/web/prisma/schema.prisma +++ b/web/prisma/schema.prisma @@ -128,6 +128,7 @@ enum AccountStatusChange { } enum NotificationType { + TEST COMMENT_STATUS_CHANGE REPLY_COMMENT_CREATED COMMUNITY_NOTE_ADDED diff --git a/web/src/actions/admin/notification.ts b/web/src/actions/admin/notification.ts index f790a93..4d4a29c 100644 --- a/web/src/actions/admin/notification.ts +++ b/web/src/actions/admin/notification.ts @@ -1,80 +1,36 @@ import { z } from 'astro/zod' -import { sumBy } from 'lodash-es' import { defineProtectedAction } from '../../lib/defineProtectedAction' import { prisma } from '../../lib/prisma' -import { sendPushNotification } from '../../lib/webPush' +import { sendNotifications } from '../../lib/sendNotifications' import { stringListOfSlugsSchemaRequired } from '../../lib/zodUtils' export const adminNotificationActions = { - webPush: { - test: defineProtectedAction({ - accept: 'form', - permissions: 'admin', - input: z.object({ - userNames: stringListOfSlugsSchemaRequired, - title: z.string().min(1).nullable(), - body: z.string().nullable(), - url: z.string().url().optional(), - }), - handler: async (input) => { - const subscriptions = await prisma.pushSubscription.findMany({ - where: { user: { name: { in: input.userNames } } }, - select: { - id: true, - endpoint: true, - p256dh: true, - auth: true, - userAgent: true, - user: { - select: { - id: true, - name: true, - }, - }, - }, - }) - - const results = await Promise.allSettled( - subscriptions.map(async (subscription) => { - const result = await sendPushNotification( - { - endpoint: subscription.endpoint, - keys: { - p256dh: subscription.p256dh, - auth: subscription.auth, - }, - }, - { - title: input.title ?? 'Test Notification', - body: input.body ?? 'This is a test push notification from KYCNot.me', - url: input.url ?? '/', - } - ) - - // If subscription is invalid, remove it from database - if (result.error && (result.error.statusCode === 410 || result.error.statusCode === 404)) { - await prisma.pushSubscription.delete({ - where: { id: subscription.id }, - }) - console.info(`Removed invalid subscription for user ${subscription.user.name}`) - } - - return result.success - }) - ) - - const successCount = sumBy(results, (r) => (r.status === 'fulfilled' && r.value ? 1 : 0)) - const failureCount = sumBy(results, (r) => (r.status === 'fulfilled' && r.value ? 0 : 1)) - const now = new Date() - return { - message: `Sent to ${successCount.toLocaleString()} devices, ${failureCount.toLocaleString()} failed. Sent at ${now.toLocaleString()}`, - totalSubscriptions: subscriptions.length, - successCount, - failureCount, - sentAt: now, - } - }, + test: defineProtectedAction({ + accept: 'form', + permissions: 'admin', + input: z.object({ + userNames: stringListOfSlugsSchemaRequired, }), - }, + handler: async (input) => { + const users = await prisma.user.findMany({ + where: { name: { in: input.userNames } }, + select: { id: true }, + }) + + const notifications = await prisma.notification.createManyAndReturn({ + data: users.map((user) => ({ + type: 'TEST', + userId: user.id, + })), + select: { id: true }, + }) + + const results = await sendNotifications(notifications.map((notification) => notification.id)) + + return { + message: `Sent to ${results.subscriptions.success.toLocaleString()} devices, ${results.subscriptions.failure.toLocaleString()} failed.`, + } + }, + }), } diff --git a/web/src/constants/notificationTypes.ts b/web/src/constants/notificationTypes.ts index 4f41018..fdfb44a 100644 --- a/web/src/constants/notificationTypes.ts +++ b/web/src/constants/notificationTypes.ts @@ -20,6 +20,11 @@ export const { icon: 'ri:notification-line', }), [ + { + id: 'TEST', + label: 'Test notification', + icon: 'ri:flask-line', + }, { id: 'COMMENT_STATUS_CHANGE', label: 'Comment status changed', diff --git a/web/src/lib/notifications.ts b/web/src/lib/notifications.ts index b9e3ea0..f13a45e 100644 --- a/web/src/lib/notifications.ts +++ b/web/src/lib/notifications.ts @@ -12,6 +12,7 @@ import type { Prisma } from '@prisma/client' export function makeNotificationTitle( notification: Prisma.NotificationGetPayload<{ select: { + id: true type: true aboutAccountStatusChange: true aboutCommentStatusChange: true @@ -87,6 +88,9 @@ export function makeNotificationTitle( user: Prisma.UserGetPayload<{ select: { id: true } }> | null ): string { switch (notification.type) { + case 'TEST': { + return `Test notification #${notification.id.toString()}` + } case 'COMMENT_STATUS_CHANGE': { if (!notification.aboutComment) return 'A comment you are watching had a status change' @@ -178,6 +182,7 @@ export function makeNotificationTitle( export function makeNotificationContent( notification: Prisma.NotificationGetPayload<{ select: { + createdAt: true type: true aboutKarmaTransaction: { select: { @@ -204,6 +209,9 @@ export function makeNotificationContent( }> ): string | null { switch (notification.type) { + case 'TEST': { + return `Created on ${notification.createdAt.toLocaleString()}` + } // TODO: [KARMA_UNLOCK] Will be added later, when karma unloks are in the database, not in the code. // case 'KARMA_UNLOCK': case 'KARMA_CHANGE': { @@ -280,6 +288,9 @@ export function makeNotificationLink( origin: string ): string | null { switch (notification.type) { + case 'TEST': { + return `${origin}/notifications` + } case 'COMMENT_STATUS_CHANGE': case 'REPLY_COMMENT_CREATED': case 'COMMUNITY_NOTE_ADDED': diff --git a/web/src/lib/postgresListenerIntegration.ts b/web/src/lib/postgresListenerIntegration.ts index 2e39a09..179de52 100644 --- a/web/src/lib/postgresListenerIntegration.ts +++ b/web/src/lib/postgresListenerIntegration.ts @@ -2,15 +2,12 @@ import { z } from 'astro/zod' import { Client } from 'pg' import { zodParseJSON } from './json' -import { makeNotificationContent, makeNotificationLink, makeNotificationTitle } from './notifications' -import { prisma } from './prisma' +import { sendNotifications } from './sendNotifications' import { getServerEnvVariable } from './serverEnvVariables' -import { sendPushNotification, type NotificationData } from './webPush' import type { AstroIntegration, HookParameters } from 'astro' const DATABASE_URL = getServerEnvVariable('DATABASE_URL') -const SITE_URL = getServerEnvVariable('SITE_URL') let pgClient: Client | null = null @@ -24,155 +21,10 @@ async function handleNotificationCreated( try { logger.info(`Processing notification with ID: ${String(notificationId)}`) - const notification = await prisma.notification.findUnique({ - where: { id: notificationId }, - select: { - id: true, - type: true, - userId: true, - aboutAccountStatusChange: true, - aboutCommentStatusChange: true, - aboutServiceVerificationStatusChange: true, - aboutSuggestionStatusChange: true, - aboutComment: { - select: { - id: true, - author: { select: { id: true } }, - status: true, - content: true, - communityNote: true, - parent: { - select: { - author: { - select: { - id: true, - }, - }, - }, - }, - service: { - select: { - slug: true, - name: true, - }, - }, - }, - }, - aboutServiceSuggestionId: true, - aboutServiceSuggestion: { - select: { - status: true, - service: { - select: { - name: true, - }, - }, - }, - }, - aboutServiceSuggestionMessage: { - select: { - id: true, - content: true, - suggestion: { - select: { - id: true, - service: { - select: { - name: true, - }, - }, - }, - }, - }, - }, - aboutEvent: { - select: { - title: true, - type: true, - service: { - select: { - slug: true, - name: true, - }, - }, - }, - }, - aboutService: { - select: { - slug: true, - name: true, - verificationStatus: true, - }, - }, - aboutKarmaTransaction: { - select: { - points: true, - action: true, - description: true, - }, - }, - user: { - select: { - id: true, - name: true, - }, - }, - }, - }) - - if (!notification) { - logger.warn(`Notification with ID ${String(notificationId)} not found`) - return - } - - const subscriptions = await prisma.pushSubscription.findMany({ - where: { userId: notification.userId }, - select: { - id: true, - endpoint: true, - p256dh: true, - auth: true, - }, - }) - - if (subscriptions.length === 0) { - logger.info(`No push subscriptions found for user ${notification.user.name}`) - return - } - const notificationData = { - title: makeNotificationTitle(notification, notification.user), - body: makeNotificationContent(notification) ?? undefined, - url: makeNotificationLink(notification, SITE_URL) ?? undefined, - } satisfies NotificationData - - const results = await Promise.allSettled( - subscriptions.map(async (subscription) => { - const result = await sendPushNotification( - { - endpoint: subscription.endpoint, - keys: { - p256dh: subscription.p256dh, - auth: subscription.auth, - }, - }, - notificationData - ) - - // Remove invalid subscriptions - if (result.error && (result.error.statusCode === 410 || result.error.statusCode === 404)) { - await prisma.pushSubscription.delete({ where: { id: subscription.id } }) - logger.info(`Removed invalid subscription for user ${notification.user.name}`) - } - - return result.success - }) - ) - - const successCount = results.filter((r) => r.status === 'fulfilled' && r.value).length - const failureCount = results.filter((r) => !(r.status === 'fulfilled' && r.value)).length + const results = await sendNotifications([notificationId], logger) logger.info( - `Push notification sent for notification ${String(notificationId)} to user ${notification.user.name}: ${String(successCount)} successful, ${String(failureCount)} failed` + `Sent push notifications for notification ${String(notificationId)} to ${String(results.subscriptions.success)} devices, ${String(results.subscriptions.failure)} failed` ) } catch (error) { logger.error(`Error processing notification ${String(notificationId)}: ${getErrorMessage(error)}`) diff --git a/web/src/lib/sendNotifications.ts b/web/src/lib/sendNotifications.ts new file mode 100644 index 0000000..2c1890c --- /dev/null +++ b/web/src/lib/sendNotifications.ts @@ -0,0 +1,185 @@ +import { sum } from 'lodash-es' + +import { makeNotificationContent, makeNotificationLink, makeNotificationTitle } from './notifications' +import { prisma } from './prisma' +import { getServerEnvVariable } from './serverEnvVariables' +import { type NotificationData, sendPushNotification } from './webPush' + +import type { AstroIntegrationLogger } from 'astro' + +const SITE_URL = getServerEnvVariable('SITE_URL') + +export async function sendNotifications( + notificationIds: number[], + logger: AstroIntegrationLogger | Console = console +) { + const notifications = await prisma.notification.findMany({ + where: { id: { in: notificationIds } }, + select: { + id: true, + type: true, + userId: true, + createdAt: true, + aboutAccountStatusChange: true, + aboutCommentStatusChange: true, + aboutServiceVerificationStatusChange: true, + aboutSuggestionStatusChange: true, + aboutComment: { + select: { + id: true, + author: { select: { id: true } }, + status: true, + content: true, + communityNote: true, + parent: { + select: { + author: { + select: { + id: true, + }, + }, + }, + }, + service: { + select: { + slug: true, + name: true, + }, + }, + }, + }, + aboutServiceSuggestionId: true, + aboutServiceSuggestion: { + select: { + status: true, + service: { + select: { + name: true, + }, + }, + }, + }, + aboutServiceSuggestionMessage: { + select: { + id: true, + content: true, + suggestion: { + select: { + id: true, + service: { + select: { + name: true, + }, + }, + }, + }, + }, + }, + aboutEvent: { + select: { + title: true, + type: true, + service: { + select: { + slug: true, + name: true, + }, + }, + }, + }, + aboutService: { + select: { + slug: true, + name: true, + verificationStatus: true, + }, + }, + aboutKarmaTransaction: { + select: { + points: true, + action: true, + description: true, + }, + }, + user: { + select: { + id: true, + name: true, + }, + }, + }, + }) + + if (notifications.length < notificationIds.length) { + const missingNotificationIds = notificationIds.filter( + (id) => !notifications.some((notification) => notification.id === id) + ) + logger.error(`Notifications with IDs ${missingNotificationIds.join(', ')} not found`) + } + + const results = await Promise.allSettled( + notifications.map(async (notification) => { + const subscriptions = await prisma.pushSubscription.findMany({ + where: { userId: notification.userId }, + select: { + id: true, + endpoint: true, + p256dh: true, + auth: true, + }, + }) + + if (subscriptions.length === 0) { + logger.info(`No push subscriptions found for user ${notification.user.name}`) + return + } + const notificationData = { + title: makeNotificationTitle(notification, notification.user), + body: makeNotificationContent(notification) ?? undefined, + url: makeNotificationLink(notification, SITE_URL) ?? undefined, + } satisfies NotificationData + + const subscriptionResults = await Promise.allSettled( + subscriptions.map(async (subscription) => { + const result = await sendPushNotification( + { + endpoint: subscription.endpoint, + keys: { + p256dh: subscription.p256dh, + auth: subscription.auth, + }, + }, + notificationData + ) + + // Remove invalid subscriptions + if (result.error && (result.error.statusCode === 410 || result.error.statusCode === 404)) { + await prisma.pushSubscription.delete({ where: { id: subscription.id } }) + logger.info(`Removed invalid subscription for user ${notification.user.name}`) + } + + return result.success + }) + ) + + return { + success: subscriptionResults.filter((r) => r.status === 'fulfilled' && r.value).length, + failure: subscriptionResults.filter((r) => !(r.status === 'fulfilled' && r.value)).length, + total: subscriptionResults.length, + } + }) + ) + + return { + subscriptions: { + success: sum(results.map((r) => (r.status === 'fulfilled' && r.value?.success) ?? 0)), + failure: sum(results.map((r) => (r.status === 'fulfilled' && r.value?.failure) ?? 0)), + total: sum(results.map((r) => (r.status === 'fulfilled' && r.value?.total) ?? 0)), + }, + notifications: { + success: results.filter((r) => r.status === 'fulfilled').length, + failure: results.filter((r) => r.status !== 'fulfilled').length, + total: results.length, + }, + } +} diff --git a/web/src/pages/admin/notifications.astro b/web/src/pages/admin/notifications.astro index 1417612..cf24864 100644 --- a/web/src/pages/admin/notifications.astro +++ b/web/src/pages/admin/notifications.astro @@ -3,8 +3,7 @@ import { Icon } from 'astro-icon/components' import { actions, isInputError } from 'astro:actions' import { groupBy, round, uniq } from 'lodash-es' -import InputSubmitButton from '../../components/InputSubmitButton.astro' -import InputText from '../../components/InputText.astro' +import Button from '../../components/Button.astro' import InputTextArea from '../../components/InputTextArea.astro' import MiniLayout from '../../layouts/MiniLayout.astro' import { cn } from '../../lib/cn' @@ -15,7 +14,7 @@ if (!Astro.locals.user?.admin) { return Astro.redirect('/access-denied') } -const testResult = Astro.getActionResult(actions.admin.notification.webPush.test) +const testResult = Astro.getActionResult(actions.admin.notification.test) const testInputErrors = isInputError(testResult?.error) ? testResult.error.fields : {} Astro.locals.banners.addIfSuccess(testResult, (data) => data.message) @@ -104,7 +103,7 @@ const stats = [

Send Test Notification

-
+ - - - - - - - +