feat: allow for bulk user deletion

apps/platform/src/users/UserController.ts

@@ -3,6 +3,7 @@ import App from '../app'
 import { ProjectState } from '../auth/AuthMiddleware'
 import UserDeleteJob from './UserDeleteJob'
 import UserPatchJob from './UserPatchJob'
+import parse from '../storage/FileStream'
 import { JSONSchemaType, validate } from '../core/validate'
 import { User, UserParams } from './User'
 import { extractQueryParams } from '../utilities'
@@ -13,6 +14,7 @@ import { SubscriptionState } from '../subscriptions/Subscription'
 import { getUserEvents } from './UserEventRepository'
 import { projectRoleMiddleware } from '../projects/ProjectService'
 import { pagedEntrancesByUser } from '../journey/JourneyRepository'
+import { removeUsers } from './UserImport'
 
 const router = new Router<
     ProjectState & { user?: User }
@@ -117,6 +119,17 @@ router.patch('/', projectRoleMiddleware('editor'), async ctx => {
     ctx.body = ''
 })
 
+router.post('/delete', async ctx => {
+    const stream = await parse(ctx)
+
+    await removeUsers({
+        project_id: ctx.state.project.id,
+        stream,
+    })
+
+    ctx.status = 204
+})
+
 const deleteUsersRequest: JSONSchemaType<string[]> = {
     $id: 'deleteUsers',
     type: 'array',

apps/platform/src/users/UserImport.ts

@@ -6,6 +6,7 @@ import { RequestError } from '../core/errors'
 import App from '../app'
 import { Chunker } from '../utilities'
 import { getList, updateListState } from '../lists/ListService'
+import UserDeleteJob from './UserDeleteJob'
 
 export interface UserImport {
     project_id: number
@@ -62,6 +63,35 @@ export const importUsers = async ({ project_id, stream, list_id }: UserImport) =
     await updateListState(list_id, { state: 'ready' })
 }
 
+export interface UserRemoval {
+    project_id: number
+    stream: FileStream
+}
+
+export const removeUsers = async ({ project_id, stream }: UserRemoval) => {
+    const options: Options = {
+        columns: true,
+        cast: true,
+        skip_empty_lines: true,
+        bom: true,
+    }
+
+    const chunker = new Chunker<UserDeleteJob>(
+        items => App.main.queue.enqueueBatch(items),
+        App.main.queue.batchSize,
+    )
+    const parser = stream.file.pipe(parse(options))
+    for await (const row of parser) {
+        const { external_id } = cleanRow(row)
+        if (!external_id) throw new RequestError('Every upload must only contain a column `external_id` which contains the identifier for that user.')
+        await chunker.add(UserDeleteJob.from({
+            project_id,
+            external_id: `${external_id}`,
+        }))
+    }
+    await chunker.flush()
+}
+
 const cleanRow = (row: Record<string, any>): Record<string, any> => {
     return Object.keys(row).reduce((acc, curr) => {
         acc[curr] = cleanCell(row[curr], curr)

apps/platform/src/users/UserRepository.ts

@@ -10,6 +10,7 @@ import { Context } from 'koa'
 import { EventPostJob } from '../jobs'
 import { Transaction } from '../core/Model'
 import App from '../app'
+import Project from '../projects/Project'
 
 export const getUser = async (id: number, projectId?: number): Promise<User | undefined> => {
     return await User.find(id, qb => {
@@ -114,13 +115,15 @@ export const aliasUser = async (projectId: number, {
     return await User.updateAndFetch(previous.id, { external_id })
 }
 
-export const createUser = async (projectId: number, { external_id, anonymous_id, data, created_at, ...fields }: UserInternalParams, trx?: Transaction) => {
+export const createUser = async (projectId: number, { external_id, anonymous_id, data, locale, created_at, ...fields }: UserInternalParams, trx?: Transaction) => {
+    const project = await Project.find(projectId)
     const user = await User.insertAndFetch({
         project_id: projectId,
         anonymous_id: anonymous_id ?? uuid(),
         external_id,
         data: data ?? {},
         devices: [],
+        locale: locale ?? project?.locale,
         created_at: created_at ? new Date(created_at) : new Date(),
         ...fields,
         version: Date.now(),
@@ -164,12 +167,6 @@ export const deleteUser = async (projectId: number, externalId: string): Promise
     const user = await getUserFromClientId(projectId, { external_id: externalId } as ClientIdentity)
     if (!user) return
 
-    // Delete the user from ClickHouse
-    await User.clickhouse().delete('project_id = {projectId: UInt32} AND id = {id: UInt32}', {
-        projectId,
-        id: user.id,
-    })
-
     // Delete the user events from the database
     await UserEvent.delete(qb => qb.where('project_id', projectId)
         .where('user_id', user.id),
@@ -180,6 +177,12 @@ export const deleteUser = async (projectId: number, externalId: string): Promise
         .where('id', user.id),
     )
 
+    // Delete the user from ClickHouse
+    await User.clickhouse().delete('project_id = {projectId: UInt32} AND id = {id: UInt32}', {
+        projectId,
+        id: user.id,
+    })
+
     // Delete the user events from ClickHouse
     await UserEvent.clickhouse().delete('project_id = {projectId: UInt32} AND user_id = {userId: UInt32}', {
         projectId,

apps/ui/public/locales/en.json

@@ -74,7 +74,7 @@
     "day_other": "{{count}} Days",
     "deeplink": "Deeplink",
     "default_locale": "Default Locale",
-    "default_locale_description": "This locale will be used as the default when creating campaigns and when a users locale does not match any available ones.",
+    "default_locale_description": "This locale will be used as the default when creating campaigns and when a user is created without a locale.",
     "defaults": "Defaults",
     "delay": "Delay",
     "delay_date_desc": "Delay until the specified date in the user's timezone.",
@@ -89,7 +89,9 @@
     "delete_key_confirmation": "Are you sure you want to archive this key? All clients using the key will immediately be unable to access the API.",
     "delete_step": "Delete Step",
     "delete_user": "Delete User",
+    "delete_users": "Delete Users",
     "delete_user_confirmation": "Are you sure you want to delete this user? All existing data will be removed.\n\nNote: If new data is sent for this user, they will be re-created with whatever data is sent.",
+    "delete_users_instructions": "Please select a CSV containing only an `external_id` column of users to be purged from the system.",
     "delivery": "Delivery",
     "delivery_rate": "Delivery Rate",
     "description": "Description",

apps/ui/src/api.ts

@@ -233,6 +233,11 @@ const api = {
         updateSubscriptions: async (projectId: number | string, userId: number | string, subscriptions: SubscriptionParams[]) => await client
             .patch(`${projectUrl(projectId)}/users/${userId}/subscriptions`, subscriptions)
             .then(r => r.data),
+        deleteImport: async (projectId: number | string, file: File) => {
+            const formData = new FormData()
+            formData.append('file', file)
+            await client.post(`${projectUrl(projectId)}/users/delete`, formData)
+        },
 
         journeys: {
             search: async (projectId: number | string, userId: number | string, params: SearchParams) => await client

apps/ui/src/views/users/Users.tsx

@@ -1,18 +1,35 @@
-import { useCallback } from 'react'
+import { useCallback, useState } from 'react'
 import { useParams } from 'react-router'
 import api from '../../api'
 import PageContent from '../../ui/PageContent'
 import { SearchTable, useSearchTableQueryState } from '../../ui/SearchTable'
 import { useRoute } from '../router'
 import { useTranslation } from 'react-i18next'
+import { Button, Modal } from '../../ui'
+import FormWrapper from '../../ui/form/FormWrapper'
+import UploadField from '../../ui/form/UploadField'
+import { TrashIcon } from '../../ui/icons'
 
 export default function UserTabs() {
     const { projectId = '' } = useParams()
     const { t } = useTranslation()
     const route = useRoute()
     const state = useSearchTableQueryState(useCallback(async params => await api.users.search(projectId, params), [projectId]))
+    const [isUploadOpen, setIsUploadOpen] = useState(false)
 
-    return <PageContent title={t('users')}>
+    const removeUsers = async (file: FileList) => {
+        await api.users.deleteImport(projectId, file[0])
+        await state.reload()
+        setIsUploadOpen(false)
+    }
+
+    return <PageContent
+        title={t('users')}
+        actions={
+            <Button icon={<TrashIcon />}
+                onClick={() => setIsUploadOpen(true)}
+                variant="destructive">{t('delete_users')}</Button>
+        }>
         <SearchTable
             {...state}
             columns={[
@@ -27,5 +44,20 @@ export default function UserTabs() {
             enableSearch
             searchPlaceholder={t('search_users')}
         />
+
+        <Modal
+            open={isUploadOpen}
+            onClose={() => setIsUploadOpen(false)}
+            title={t('delete_users')}>
+            <FormWrapper<{ file: FileList }>
+                onSubmit={async (form) => await removeUsers(form.file)}
+                submitLabel={t('delete')}
+            >
+                {form => <>
+                    <p>{t('delete_users_instructions')}</p>
+                    <UploadField form={form} name="file" label={t('file')} required />
+                </>}
+            </FormWrapper>
+        </Modal>
     </PageContent>
 }