From 6ccef2e8334d9f1ea78adb5eef90960ab2a18a44 Mon Sep 17 00:00:00 2001
From: aliamerj <aliamer19ali@gmail.com>
Date: Sat, 17 May 2025 21:33:33 +0300
Subject: [PATCH 1/2] set up dropbox testing and fix bug

---
 DataSource/Dropbox/setDropboxConnection.ts    |   1 -
 app/api/connections/dropbox/callback/route.ts |   2 +-
 app/layout.tsx                                |   8 +-
 .../ConfigConnection/ConfigConnection.tsx     |  18 ++-
 cypress.config.ts                             |   6 +-
 cypress/e2e/connections.cy.ts                 |   4 +-
 cypress/e2e/dropbox.cy.ts                     | 133 ++++++++++++++++++
 cypress/support/commands.ts                   |   8 +-
 fileProcessors/index.ts                       |  22 ++-
 mocks/handlers.ts                             |  85 ++++++++++-
 10 files changed, 264 insertions(+), 23 deletions(-)
 create mode 100644 cypress/e2e/dropbox.cy.ts

diff --git a/DataSource/Dropbox/setDropboxConnection.ts b/DataSource/Dropbox/setDropboxConnection.ts
index a7fd848..fa72429 100644
--- a/DataSource/Dropbox/setDropboxConnection.ts
+++ b/DataSource/Dropbox/setDropboxConnection.ts
@@ -3,7 +3,6 @@ import { connections } from "@/db/schemas/connections";
 import { eq } from "drizzle-orm";
 import { connectionConfig } from "../utils";
 
-
 export const setDropboxConnection = async (formData: FormData) => {
   const config = connectionConfig.safeParse({
     connectionId: formData.get("connectionId"),
diff --git a/app/api/connections/dropbox/callback/route.ts b/app/api/connections/dropbox/callback/route.ts
index 3b68d5c..c14051f 100644
--- a/app/api/connections/dropbox/callback/route.ts
+++ b/app/api/connections/dropbox/callback/route.ts
@@ -104,6 +104,6 @@ export async function GET(request: Request) {
     }
     return NextResponse.redirect(`${process.env.NEXTAUTH_URL}/connections`);
   } catch (error) {
-    return NextResponse.json({ error: 'Failed to authenticate' }, { status: 500 });
+    return NextResponse.json({ error: 'Failed to authenticate' }, { status: 401 });
   }
 }
diff --git a/app/layout.tsx b/app/layout.tsx
index de90952..2b53cfb 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -30,11 +30,9 @@ export const metadata: Metadata = {
 };
 
 if (process.env.NEXT_PUBLIC_APP_ENV === 'TEST') {
-  if (!(global as any).__MSW_SERVER_STARTED__) {
-    console.log('🔧 MSW mock server starting...');
-    mockServer.listen();
-    (global as any).__MSW_SERVER_STARTED__ = true;
-  }
+  console.log('🔧🔧🔧 MSW mock server starting...🔧🔧🔧');
+  mockServer.listen();
+  (global as any).__MSW_SERVER_STARTED__ = true;
 }
 
 export default async function Layout({ children }: { children: ReactNode }) {
diff --git a/components/ConfigConnection/ConfigConnection.tsx b/components/ConfigConnection/ConfigConnection.tsx
index 94e46de..f49ee14 100644
--- a/components/ConfigConnection/ConfigConnection.tsx
+++ b/components/ConfigConnection/ConfigConnection.tsx
@@ -32,6 +32,7 @@ type ConnectionProps = {
 export const ConfigConnection = ({ connection, directory, status, open, setOpen, showPicker }: ConnectionProps) => {
   const [isConfigSet, setIsConfigSet] = useState(connection.isConfigSet)
   const [pending, startTransition] = useTransition()
+  const isTest = process.env.NEXT_PUBLIC_APP_ENV === 'TEST'
 
   // Service-specific text configuration
   const serviceLabels = {
@@ -55,9 +56,15 @@ export const ConfigConnection = ({ connection, directory, status, open, setOpen,
 
   const handleSetConfig = (data: FormData) => {
     data.set("connectionId", connection.id)
-    data.set("folderName", directory.name)
     data.set("service", connection.service)
+
     if (directory?.id) data.set("folderId", directory.id)
+    if (isTest) {
+      const folderName = data.get("folderName")
+      data.set("folderId", folderName || "")
+    } else {
+      data.set("folderName", directory.name)
+    }
     startTransition(async () => {
       const setConfig = async () => {
         try {
@@ -121,11 +128,12 @@ export const ConfigConnection = ({ connection, directory, status, open, setOpen,
             <label className="block text-sm font-medium">Folder</label>
             <div className="flex items-center gap-2">
               <Input
-                value={directory?.name || connection.folderName || ""}
+                name='folderName'
+                value={isTest ? undefined : directory?.name || connection.folderName || ""}
                 placeholder={getServiceLabel('placeholder')}
-                disabled={connection.isSyncing || connection.isConfigSet}
-                onClick={showPicker} type='button'
-                readOnly
+                onClick={isTest ? undefined : showPicker}
+                type={isTest ? 'text' : 'button'}
+                readOnly={!isTest}
               />
               <Button onClick={showPicker} type="button" disabled={connection.isSyncing || connection.isConfigSet}  >
                 {getServiceLabel('selectButton')}
diff --git a/cypress.config.ts b/cypress.config.ts
index 09c5cdf..a1cd857 100644
--- a/cypress.config.ts
+++ b/cypress.config.ts
@@ -11,9 +11,9 @@ const qdrant_collection_name = "documents"
 const qdrantCLient = new QdrantClient({ url: process.env.QDRANT_DB_URL!, apiKey: process.env.QDRANT_DB_KEY });
 
 export default defineConfig({
-    retries: {
-      runMode: 2,
-      openMode: 0
+  retries: {
+    runMode: 2,
+    openMode: 0
   },
   chromeWebSecurity: false,
   watchForFileChanges: false,
diff --git a/cypress/e2e/connections.cy.ts b/cypress/e2e/connections.cy.ts
index 2d018c8..443c5fb 100644
--- a/cypress/e2e/connections.cy.ts
+++ b/cypress/e2e/connections.cy.ts
@@ -1,6 +1,6 @@
 import { connections } from "@/db/schemas/connections"
 
-describe.skip('connect to google drive', () => {
+describe('connect to google drive', () => {
   const fakeUser = {
     name: "test man",
     email: "tester@dcup.dev",
@@ -45,7 +45,7 @@ describe.skip('connect to google drive', () => {
   })
 
 
-  it('connect and config with no settings and delete ( 0 file ) , check the database', () => {
+  it.only('connect and config with no settings and delete ( 0 file ) , check the database', () => {
     cy.task('getConnections', { email: fakeUser.email })
       .then(res => {
         const { conns } = res as { conns: typeof connections.$inferSelect[] }
diff --git a/cypress/e2e/dropbox.cy.ts b/cypress/e2e/dropbox.cy.ts
new file mode 100644
index 0000000..ae83643
--- /dev/null
+++ b/cypress/e2e/dropbox.cy.ts
@@ -0,0 +1,133 @@
+import { ConnectionTable, ProcessedFilesTable } from "@/db/schemas/connections"
+
+
+interface FileConnectionQuery extends ConnectionTable {
+  files: ProcessedFilesTable[]
+}
+const fakeUser = {
+  name: "test man",
+  email: "tester@dcup.dev",
+  provider: "google",
+  image: `https://via.placeholder.com/200/7732bb/c0392b.png?text=tester`,
+  plan: "OS"
+}
+describe.only("Dropbox connection UI Testing", () => {
+  beforeEach(() => {
+
+    // create Collection
+    cy.visit('/')
+    cy.wait(1000)
+    cy.task("createCollection")
+
+    // login 
+    cy.loginNextAuth(fakeUser)
+      .visit('/')
+      .location('pathname')
+      .should('eq', '/')
+      .visit('/')
+      .location('pathname')
+      .should('eq', '/')
+
+    // create new connection 
+    cy.intercept("GET", "https://www.dropbox.com/oauth2/authorize?response_type=code&client_id=reqn50m9i1xnewd&redirect_uri=http://localhost:3000/api/connections/dropbox/callback&token_access_type=offline&scope=files.metadata.read%20files.content.read%20email%20openid%20account_info.read&force_reapprove=true", (res) => {
+      res.reply(200, "OK")
+    }).as("dropboxAuth")
+
+    cy.visit('/connections/new')
+      .get('[data-test="connection-card"]')
+      .contains('Dropbox', { matchCase: false })
+      .get('[data-test="btn-dropbox"]')
+      .click();
+
+    cy.wait("@dropboxAuth")
+    cy.request("/api/connections/dropbox/callback?code=FAKE_CODE")
+      .then(res => cy.log(`status: ${res.status}`))
+  })
+
+  afterEach(() => {
+    cy.task('deleteUser', { email: fakeUser.email })
+    cy.task("deleteCollection")
+  })
+
+  it("It should handle file upload, addition, and removal with database synchronization", () => {
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.visit('/connections')
+          .get(`[data-test="btn-config-${conns[0].identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+    cy.wait(5_000)
+    // Check 
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 3 }] })
+
+        cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/invo.pdf")
+        cy.get('[data-test="processedFile"]').should('contain.text', 1)
+        cy.get('[data-test="processedPage"]').should('contain.text', 3)
+
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf/sample.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+
+    cy.wait(5_000)
+    //Check
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({
+          conn,
+          source: "DROPBOX",
+          files: [{ name: "sample.pdf", totalPages: 1 }, { name: "invo.pdf", totalPages: 3 }]
+        })
+
+        cy.get('[data-test="folderName"]').should('contain.text', '_TEST_/invo.pdf/sample.pdf')
+        cy.get('[data-test="processedFile"]').should('contain.text', 2)
+        cy.get('[data-test="processedPage"]').should('contain.text', 4)
+
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/sample.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+
+    cy.wait(5_000)
+    // check
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "sample.pdf", totalPages: 1 }] })
+      })
+
+    cy.get('[data-test="folderName"]').should('contain.text', '_TEST_/sample.pdf')
+    cy.get('[data-test="processedFile"]').should('contain.text', 1)
+    cy.get('[data-test="processedPage"]').should('contain.text', 1)
+  })
+})
diff --git a/cypress/support/commands.ts b/cypress/support/commands.ts
index 1180852..fc2f254 100644
--- a/cypress/support/commands.ts
+++ b/cypress/support/commands.ts
@@ -53,7 +53,7 @@ Cypress.Commands.add("uploadFiles", ({ files }) => {
     .click({ force: true })
 })
 
-Cypress.Commands.add("checkIndexedFiles", ({ conn, files }) => {
+Cypress.Commands.add("checkIndexedFiles", ({ conn, files, source }) => {
   Cypress.log({
     displayName: "INDEX Files",
     message: files.map(f => `name:${f.name}, totalPages: ${f.totalPages}`),
@@ -67,7 +67,7 @@ Cypress.Commands.add("checkIndexedFiles", ({ conn, files }) => {
       expect(ps.length).eq(conn.files[index].chunksIds.length)
       ps.map(p => {
         expect(p.payload._document_id).eq(file.name)
-        expect(p.payload._metadata.source).eq('DIRECT_UPLOAD')
+        expect(p.payload._metadata.source).eq( source ?? 'DIRECT_UPLOAD')
       })
     })
   })
@@ -141,12 +141,14 @@ Cypress.Commands.add("uploadFileWithApi", ({ fileName, apiKey, url, method, link
 // -- This will overwrite an existing command --
 // Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })
 //
+//
+//
 declare global {
   namespace Cypress {
     interface Chainable {
       loginNextAuth({ name, email, image }: { name: string, email: string, image: string }): Chainable<Cookie>
       uploadFiles({ files }: { files: string[] }): Chainable<void>
-      checkIndexedFiles({ conn, files }: { conn: FileConnectionQuery, files: { name: string, totalPages: number }[] }): Chainable<void>
+      checkIndexedFiles({ conn, files, source }: { conn: FileConnectionQuery, files: { name: string, totalPages: number }[], source?:string }): Chainable<void>
       uploadFileWithApi({ fileName, apiKey, response, url }: UploadRequest): Chainable<void>
       // login(email: string, password: string): Chainable<void>
       // drag(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
diff --git a/fileProcessors/index.ts b/fileProcessors/index.ts
index 4b8ff6a..529386e 100644
--- a/fileProcessors/index.ts
+++ b/fileProcessors/index.ts
@@ -6,12 +6,11 @@ import { getTitleAndSummary, vectorizeText } from "@/openAi";
 import { qdrant_collection_name, qdrantCLient } from "@/qdrant";
 import { publishProgress } from "@/events";
 import { connections, processedFiles } from "@/db/schemas/connections";
-import { eq } from "drizzle-orm";
+import { and, eq } from "drizzle-orm";
 import { getFileContent } from "./connectors";
 import { processPdfLink, processPdfBuffer } from "./Files/pdf";
 import { TQueue } from "@/workers/queues/jobs/processFiles.job";
 
-
 export type FileContent = {
   name: string,
   pages: PageContent[],
@@ -83,9 +82,28 @@ export const directProcessFiles = async ({ files, metadata, service, connectionI
 export const connectionProcessFiles = async ({ connectionId, service, pageLimit, fileLimit }: TQueue) => {
   const connection = await databaseDrizzle.query.connections.findFirst({
     where: (c, ops) => ops.eq(c.id, connectionId),
+    with: {
+      files: true,
+    }
   })
   if (!connection) return;
   const filesContent = await getFileContent(connection)
+
+  const newFileNames = new Set(filesContent.map(f => f.name));
+  const filesToRemove = connection.files.filter(f => !newFileNames.has(f.name));
+
+  for (const file of filesToRemove) {
+    await databaseDrizzle
+      .delete(processedFiles)
+      .where(and(
+        eq(processedFiles.connectionId, connection.id),
+        eq(processedFiles.name, file.name)
+      ))
+
+    await qdrantCLient.delete(qdrant_collection_name, {
+      points: file.chunksIds,
+    })
+  }
   return processFiles(filesContent, service, connectionId, pageLimit, fileLimit, 0, 0)
 }
 
diff --git a/mocks/handlers.ts b/mocks/handlers.ts
index d8f3db1..f8dd904 100644
--- a/mocks/handlers.ts
+++ b/mocks/handlers.ts
@@ -1,6 +1,89 @@
+import fs from 'node:fs'
 import { http, HttpResponse } from 'msw'
-
+import path from 'path';
+type DropboxFile = {
+  path: string,
+  recursive: boolean
+}
 export const handlers = [
+  http.post("https://content.dropboxapi.com/2/files/download", async ({ request }) => {
+    // parse the Dropbox-API-Arg header
+    const header = request.headers.get('Dropbox-API-Arg') || '{}'
+    const { path: filePathLower } = JSON.parse(header) as { path: string }
+    const filename = filePathLower.replace(/^\//, '')
+
+    // resolve it under cypress/fixtures
+    const fullPath = path.resolve(process.cwd(), 'cypress', 'fixtures', filename)
+    const buffer = fs.readFileSync(fullPath)
+
+    return HttpResponse.arrayBuffer(buffer, {
+      headers: {
+        'Content-Type': 'application/pdf'
+      },
+    })
+
+  }),
+  http.post("https://api.dropboxapi.com/2/files/list_folder", async (res) => {
+    const body = (await res.request.json()) as DropboxFile
+    const files = body.path.split('/').slice(1)
+    return HttpResponse.json({
+      cursor: "ZtkX9_EHj3x7PMkVuFIhwKYXEpwpLwyxp9vMKomUhllil9q7eWiAu",
+      has_more: false,
+      entries: files.map(f => ({
+        ".tag": "file",
+        "client_modified": "2015-05-12T15:50:38Z",
+        "content_hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+        "file_lock_info": {
+          "created": "2015-05-12T15:50:38Z",
+          "is_lockholder": true,
+          "lockholder_name": "Imaginary User"
+        },
+        "has_explicit_shared_members": false,
+        "id": "id:a4ayc_80_OEAAAAAAAAAXw",
+        "is_downloadable": true,
+        "name": `${f}`,
+        "path_display": `/${f}`,
+        "path_lower": `/${f}`,
+        "property_groups": [
+          {
+            "fields": [
+              {
+                "name": "Security Policy",
+                "value": "Confidential"
+              }
+            ],
+            "template_id": "ptid:1a5n2i6d3OYEAAAAAAAAAYa"
+          }
+        ],
+        "rev": "a1c10ce0dd78",
+        "server_modified": "2015-05-12T15:50:38Z",
+        "sharing_info": {
+          "modified_by": "dbid:AAH4f99T0taONIb-OurWxbNQ6ywGRopQngc",
+          "parent_shared_folder_id": "84528192421",
+          "read_only": true
+        },
+        "size": 7212
+      })
+      )
+    }, { status: 200 })
+  }),
+  http.post("https://api.dropboxapi.com/2/users/get_current_account", () => {
+    return HttpResponse.json({
+      uid: '987654321',
+      email: "testing@gmail.com",
+    }, { status: 200 })
+  }),
+  http.post("https://api.dropboxapi.com/oauth2/token", () => {
+    return HttpResponse.json({
+      access_token: 'mocked_access_token_123',
+      token_type: 'bearer',
+      expires_in: 14400,
+      refresh_token: 'mocked_refresh_token_456',
+      scope: 'files.metadata.read',
+      uid: '987654321',
+      account_id: 'dbid:ABCDEFGH'
+    }, { status: 200 })
+  }),
   http.post("https://models.inference.ai.azure.com/embeddings", () => {
     return HttpResponse.json({
       data: [

From 1dbe669b3d0dd633cac7b2f14109eecbc86d1cb2 Mon Sep 17 00:00:00 2001
From: aliamerj <aliamer19ali@gmail.com>
Date: Sun, 18 May 2025 13:25:24 +0300
Subject: [PATCH 2/2] test and fix dropbox tests

Signed-off-by: aliamerj <aliamer19ali@gmail.com>
---
 .github/workflows/devs.yml         |   2 +
 app/layout.tsx                     |   4 +-
 cypress.config.ts                  |   2 +-
 cypress/e2e/connections.cy.ts      |   5 +-
 cypress/e2e/directUpload.cy.ts     |   5 +-
 cypress/e2e/dropbox.cy.ts          | 399 ++++++++++++++++++++++++++++-
 cypress/e2e/fileProcessors.cy.ts   | 101 --------
 fileProcessors/connectors/index.ts |   3 +-
 fileProcessors/index.ts            |  17 ++
 mocks/handlers.ts                  |   6 +-
 10 files changed, 422 insertions(+), 122 deletions(-)
 delete mode 100644 cypress/e2e/fileProcessors.cy.ts

diff --git a/.github/workflows/devs.yml b/.github/workflows/devs.yml
index cc32d98..89082c6 100644
--- a/.github/workflows/devs.yml
+++ b/.github/workflows/devs.yml
@@ -37,6 +37,8 @@ jobs:
          NEXT_PUBLIC_APP_ENV=TEST
          DCUP_PARSER=http://localhost:9000
          NEXTAUTH_URL=http://localhost:3000
+         DROPBOX_CLIENT_ID=xxxxxxx
+         DROPBOX_CLIENT_SECRET=xxxxx
          REDIS_DB_ADDRESS=localhost
          REDIS_DB_PORT=6379
          REDIS_DB_PASSWORD=
diff --git a/app/layout.tsx b/app/layout.tsx
index 2b53cfb..e2f5e2e 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -31,7 +31,9 @@ export const metadata: Metadata = {
 
 if (process.env.NEXT_PUBLIC_APP_ENV === 'TEST') {
   console.log('🔧🔧🔧 MSW mock server starting...🔧🔧🔧');
-  mockServer.listen();
+  mockServer.listen({
+    onUnhandledRequest: 'bypass',
+  });
   (global as any).__MSW_SERVER_STARTED__ = true;
 }
 
diff --git a/cypress.config.ts b/cypress.config.ts
index a1cd857..fbbe974 100644
--- a/cypress.config.ts
+++ b/cypress.config.ts
@@ -12,7 +12,7 @@ const qdrantCLient = new QdrantClient({ url: process.env.QDRANT_DB_URL!, apiKey:
 
 export default defineConfig({
   retries: {
-    runMode: 2,
+    runMode: 3,
     openMode: 0
   },
   chromeWebSecurity: false,
diff --git a/cypress/e2e/connections.cy.ts b/cypress/e2e/connections.cy.ts
index 443c5fb..c251082 100644
--- a/cypress/e2e/connections.cy.ts
+++ b/cypress/e2e/connections.cy.ts
@@ -10,9 +10,9 @@ describe('connect to google drive', () => {
   }
 
   beforeEach(() => {
-    // login
     cy.visit('/')
     cy.wait(1000)
+    cy.task("createCollection")
 
     // login 
     cy.loginNextAuth(fakeUser)
@@ -42,10 +42,11 @@ describe('connect to google drive', () => {
 
   afterEach(() => {
     cy.task('deleteUser', { email: fakeUser.email })
+    cy.task("deleteCollection")
   })
 
 
-  it.only('connect and config with no settings and delete ( 0 file ) , check the database', () => {
+  it('connect and config with no settings and delete ( 0 file ) , check the database', () => {
     cy.task('getConnections', { email: fakeUser.email })
       .then(res => {
         const { conns } = res as { conns: typeof connections.$inferSelect[] }
diff --git a/cypress/e2e/directUpload.cy.ts b/cypress/e2e/directUpload.cy.ts
index c70cbe5..8e8413c 100644
--- a/cypress/e2e/directUpload.cy.ts
+++ b/cypress/e2e/directUpload.cy.ts
@@ -250,9 +250,8 @@ describe("Direct Upload UI", () => {
     cy.get('[data-test="folderName"]').should('contain.text', "*")
     cy.get('[data-test="processedFile"]').should('contain.text', 1)
     cy.get('[data-test="processedPage"]').should('contain.text', 2)
-
-    // remove the only pdf file 
-    // with 3 pages and now we have space it should processed smail one
+   
+    // remove the only stored pdf file  
     cy.get('[data-test="btn-config"]')
       .click()
 
diff --git a/cypress/e2e/dropbox.cy.ts b/cypress/e2e/dropbox.cy.ts
index ae83643..69387cd 100644
--- a/cypress/e2e/dropbox.cy.ts
+++ b/cypress/e2e/dropbox.cy.ts
@@ -11,13 +11,11 @@ const fakeUser = {
   image: `https://via.placeholder.com/200/7732bb/c0392b.png?text=tester`,
   plan: "OS"
 }
-describe.only("Dropbox connection UI Testing", () => {
+describe("Dropbox connection UI Testing", () => {
   beforeEach(() => {
-
     // create Collection
     cy.visit('/')
-    cy.wait(1000)
-    cy.task("createCollection")
+      .task("createCollection")
 
     // login 
     cy.loginNextAuth(fakeUser)
@@ -29,9 +27,13 @@ describe.only("Dropbox connection UI Testing", () => {
       .should('eq', '/')
 
     // create new connection 
-    cy.intercept("GET", "https://www.dropbox.com/oauth2/authorize?response_type=code&client_id=reqn50m9i1xnewd&redirect_uri=http://localhost:3000/api/connections/dropbox/callback&token_access_type=offline&scope=files.metadata.read%20files.content.read%20email%20openid%20account_info.read&force_reapprove=true", (res) => {
-      res.reply(200, "OK")
-    }).as("dropboxAuth")
+    cy.intercept({
+      method: 'GET',
+      url: /https:\/\/www\.dropbox\.com\/oauth2\/authorize\?.*/,
+    },
+      (res) => {
+        res.reply(200, "OK")
+      }).as("dropboxAuth")
 
     cy.visit('/connections/new')
       .get('[data-test="connection-card"]')
@@ -42,6 +44,7 @@ describe.only("Dropbox connection UI Testing", () => {
     cy.wait("@dropboxAuth")
     cy.request("/api/connections/dropbox/callback?code=FAKE_CODE")
       .then(res => cy.log(`status: ${res.status}`))
+    cy.visit("/connections")
   })
 
   afterEach(() => {
@@ -49,12 +52,11 @@ describe.only("Dropbox connection UI Testing", () => {
     cy.task("deleteCollection")
   })
 
-  it("It should handle file upload, addition, and removal with database synchronization", () => {
+  it("should connect to dropbox handle 1 file , addition other file, and removal with database synchronization", () => {
     cy.task('getConnections', { email: fakeUser.email })
       .then(res => {
         const { conns } = res as { conns: ConnectionTable[] }
-        cy.visit('/connections')
-          .get(`[data-test="btn-config-${conns[0].identifier}"]`)
+        cy.get(`[data-test="btn-config-${conns[0].identifier}"]`)
           .click()
           .get('input[name="folderName"]')
           .clear()
@@ -130,4 +132,381 @@ describe.only("Dropbox connection UI Testing", () => {
     cy.get('[data-test="processedFile"]').should('contain.text', 1)
     cy.get('[data-test="processedPage"]').should('contain.text', 1)
   })
+
+  it('should update and clear connection metadata for indexed files', () => {
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.get(`[data-test="btn-config-${conns[0].identifier}"]`)
+          .click()
+          .get('input[name="identifier"]')
+          .clear()
+          .type("testing_connection")
+          .get('textarea[name="metadata"]')
+          .clear()
+          .type('{"job": "my test files"}', { parseSpecialCharSequences: false })
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+
+    cy.wait(5_000)
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.identifier).eq("testing_connection")
+        expect(conn.metadata).eq('{"job": "my test files"}')
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 3 }] })
+
+        cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/invo.pdf")
+        cy.get('[data-test="processedFile"]').should('contain.text', 1)
+        cy.get('[data-test="processedPage"]').should('contain.text', 3)
+
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('input[name="identifier"]')
+          .clear()
+          .type("other_testing_connection")
+          .get('textarea[name="metadata"]')
+          .clear()
+          .type('{"job": "other tests"}', { parseSpecialCharSequences: false })
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+
+    cy.wait(5_000)
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.identifier).eq("other_testing_connection")
+        expect(conn.metadata).eq('{"job": "other tests"}')
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 3 }] })
+
+        cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/invo.pdf")
+        cy.get('[data-test="processedFile"]').should('contain.text', 1)
+        cy.get('[data-test="processedPage"]').should('contain.text', 3)
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('textarea[name="metadata"]')
+          .clear()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+
+    cy.wait(5_000)
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.identifier).eq("other_testing_connection")
+        expect(conn.metadata).eq('{}')
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 3 }] })
+      })
+    cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/invo.pdf")
+    cy.get('[data-test="processedFile"]').should('contain.text', 1)
+    cy.get('[data-test="processedPage"]').should('contain.text', 3)
+  })
+
+  it('should enforce page limits during file operations and maintain constraints', () => {
+    // Upload 1 pdf  with 3 pages, it should process only 2 
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.get(`[data-test="btn-config-${conns[0].identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf')
+          .get('input[name="pageLimit"]')
+          .type('2')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+    cy.wait(5_000)
+    // Check 
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).eq(2)
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 2 }] })
+
+        cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+          .then(points => {
+            expect(points).to.be.greaterThan(0)
+          })
+
+        // check the UI
+        cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/invo.pdf")
+        cy.get('[data-test="processedFile"]').should('contain.text', 1)
+        cy.get('[data-test="processedPage"]').should('contain.text', 2)
+
+        // add new pdf file with 1 page, it should not index it or store it 
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/invo.pdf/sample.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+
+    cy.wait(5_000)
+    //Check
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).eq(2)
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 2 }] })
+
+        cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+          .then(points => {
+            expect(points).to.be.greaterThan(0)
+          })
+        cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+          .then(points => {
+            expect(points).eq(0)
+          })
+        // check the UI
+        cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/invo.pdf/sample.pdf")
+        cy.get('[data-test="processedFile"]').should('contain.text', 1)
+        cy.get('[data-test="processedPage"]').should('contain.text', 2)
+
+        // remove the only pdf file 
+        // with 3 pages and now we have space it should processed smail one
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/sample.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+    cy.wait(5_000)
+
+    // check
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).eq(2)
+        expect(conn.limitFiles).to.be.null
+        expect(conn.files.length).eq(1)
+
+        cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+          .then(points => {
+            expect(points).eq(0)
+          })
+        cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+          .then(points => {
+            expect(points).greaterThan(0)
+          })
+      })
+    // check the UI
+    cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/sample.pdf")
+    cy.get('[data-test="processedFile"]').should('contain.text', 1)
+    cy.get('[data-test="processedPage"]').should('contain.text', 1)
+  })
+
+  it('should distribute page processing across files to respect page limits', () => {
+    // Upload 1 pdf  with 1 page, it should process only 1
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.get(`[data-test="btn-config-${conns[0].identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/sample.pdf')
+          .get('input[name="pageLimit"]')
+          .type('2')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+    cy.wait(5_000)
+
+    // Check 
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).eq(2)
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "sample.pdf", totalPages: 1 }] })
+
+        cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+          .then(points => {
+            expect(points).to.be.greaterThan(0)
+          })
+        cy.wait(1000)
+
+        // check the UI
+        cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/sample.pdf")
+        cy.get('[data-test="processedFile"]').should('contain.text', 1)
+        cy.get('[data-test="processedPage"]').should('contain.text', 1)
+
+        // add new pdf file with 3 pages, it should process 2 from this file 
+        cy.get(`[data-test="btn-config-${conn.identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/sample.pdf/invo.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+    cy.wait(5000)
+
+    //Check
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).eq(2)
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 1 }, { name: "sample.pdf", totalPages: 1 }] })
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+      .then(points => {
+        expect(points).to.be.greaterThan(0)
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+      .then(points => {
+        expect(points).to.be.greaterThan(0)
+      })
+    cy.wait(1000)
+
+    // check the UI
+
+    cy.get('[data-test="folderName"]').should('contain.text', "_TEST_/sample.pdf")
+    cy.get('[data-test="processedFile"]').should('contain.text', 2)
+    cy.get('[data-test="processedPage"]').should('contain.text', 2)
+
+    // Delete connection
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.get(`[data-test="btn-delete-${conns[0].identifier}"]`)
+          .click()
+          .get(`[data-test="delete-connection-btn"]`)
+          .click()
+
+        cy.get('[data-test="delete-connection-btn"]', { timeout: 15000 })
+          .should('not.exist');
+        cy.get(`[data-test="btn-delete-${conns[0].identifier}"]`, { timeout: 15000 })
+          .should('not.exist')
+      })
+    cy.wait(1000)
+
+    // check
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        expect(conns).to.have.length(0);
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+      .then(points => {
+        expect(points).eq(0)
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+      .then(points => {
+        expect(points).eq(0)
+      })
+  })
+
+  it("should remove all files that related to the connection when user delete the connection", () => {
+    // Upload 2 pdf  
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.get(`[data-test="btn-config-${conns[0].identifier}"]`)
+          .click()
+          .get('input[name="folderName"]')
+          .clear()
+          .type('_TEST_/sample.pdf/invo.pdf')
+          .get(`[data-test="btn-config-connection"]`)
+          .click()
+      })
+    cy.wait(5000)
+
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.service).eq("DROPBOX")
+        expect(conn.metadata).eq("{}")
+        expect(conn.limitPages).to.be.null
+        expect(conn.limitFiles).to.be.null
+        cy.checkIndexedFiles({ conn, source: "DROPBOX", files: [{ name: "invo.pdf", totalPages: 3 }, { name: "sample.pdf", totalPages: 1 }] })
+      })
+    cy.wait(1000)
+
+    // check
+    cy.task("getConnection", { email: fakeUser.email })
+      .then(({ conns }: any) => {
+        const conn = (conns as FileConnectionQuery[])[0]
+        expect(conn.files.length).eq(2)
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+      .then(points => {
+        expect(points).greaterThan(0)
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+      .then(points => {
+        expect(points).greaterThan(0)
+      })
+
+    // Delete connection
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        cy.get(`[data-test="btn-delete-${conns[0].identifier}"]`)
+          .click()
+          .get(`[data-test="delete-connection-btn"]`)
+          .click()
+
+        cy.get('[data-test="delete-connection-btn"]', { timeout: 15000 })
+          .should('not.exist');
+        cy.get(`[data-test="btn-delete-${conns[0].identifier}"]`, { timeout: 15000 })
+          .should('not.exist')
+      })
+    cy.wait(1000)
+
+    // check
+    cy.task('getConnections', { email: fakeUser.email })
+      .then(res => {
+        const { conns } = res as { conns: ConnectionTable[] }
+        expect(conns).to.have.length(0);
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "invo.pdf" })
+      .then(points => {
+        expect(points).eq(0)
+      })
+    cy.task("getPointsNumberByFileName", { fileName: "sample.pdf" })
+      .then(points => {
+        expect(points).eq(0)
+      })
+  })
 })
diff --git a/cypress/e2e/fileProcessors.cy.ts b/cypress/e2e/fileProcessors.cy.ts
deleted file mode 100644
index 5153378..0000000
--- a/cypress/e2e/fileProcessors.cy.ts
+++ /dev/null
@@ -1,101 +0,0 @@
-
-describe.skip('processing file with Direct upload using APIs', () => {
-  const fakeUser = {
-    name: "test man",
-    email: "tester@dcup.dev",
-    provider: "google",
-    image: `https://via.placeholder.com/200/7732bb/c0392b.png?text=tester`,
-    plan: "OS"
-  }
-
-  beforeEach(() => {
-    cy.visit("/")
-  })
-  afterEach(() => {
-    cy.task('deleteUser', { email: fakeUser.email })
-  })
-
-
-  it("should process files successfully when all conditions are met without metadata", () => {
-    //   cy.task('addNewUser', fakeUser).then(user => {
-    //     const u = user as typeof users.$inferSelect;
-    //     cy.task('createApiKey', { id: u.id }).then(key => {
-    //       const formData = new FormData();
-    //       formData.append('links', 'test.pdf');
-    //       cy.request({
-    //         method: "POST",
-    //         url: "/api/upload",
-    //         headers: {
-    //           "Authorization": `Bearer ${key}`
-    //         },
-    //         body: formData
-    //       }).then(res => cy.log(res.body.toString()))
-    //     });
-    //   });
-    // });
-    //
-    // const data = new FormData();
-    // cy.task('addNewUser', fakeUser).then(user => {
-    //   const u = user as typeof users.$inferSelect;
-    //   cy.task('createApiKey', { id: u.id }).then(key => {
-    //     cy.fixture("invo.pdf", 'binary')
-    //       .then((file) => Cypress.Blob.binaryStringToBlob(file))
-    //       .then((blob) => {
-    //         cy.window().then(win => {
-    //           const xhr = new win.XMLHttpRequest();
-    //           data.set("files", blob, "test.pdf");
-    //           xhr.open("POST", "/api/upload");
-    //           xhr.setRequestHeader("Authorization", `Bearer ${key}`);
-    //           xhr.send(data);
-    //         }).then(response => {
-    //           expect(response.status).to.eq(200);
-    //         })
-    //       })
-    //   });
-    // });
-  });
-
-  // create user with api key
-  // create pdf file to upload
-  // call the api 
-  // check the database 
-  // check the vector database
-  it("should process files successfully when all conditions are met with metadata", () => {
-    // create user with api key
-    // create pdf file to upload
-    // call the api 
-    // check the database 
-    // check the vector database
-  })
-
-  it("should process files successfully and remove indexed files when  when user remove one of them", () => {
-    // create user with api key
-    // create 2 pdfs files to upload
-    // call the api 
-    // check the database 
-    // check the vector database
-    // remove one the pdfs 
-    // check the database
-    // check the vector db 
-  })
-  it("should add new files to the connection when we add new file to the connection", () => {
-    // create user with api key
-    // create pdf file to upload
-    // call the api 
-    // check the database 
-    // check the vector database
-    // add new pdf to the same connection
-    // check the vector database 
-    // check the database
-  })
-  it("should remove all file from this connection when get deleted", () => {
-    // create user with api key 
-    // create 2 pdfs files to upload
-    // call the api 
-    // check the db 
-    // check the vector database
-    // remove the connection 
-    // check the db 
-    // check the vector database
-  })
-})
diff --git a/fileProcessors/connectors/index.ts b/fileProcessors/connectors/index.ts
index 1ba00d4..01b0ab8 100644
--- a/fileProcessors/connectors/index.ts
+++ b/fileProcessors/connectors/index.ts
@@ -84,7 +84,8 @@ export const setConnectionToProcess = async (formData: FormData): Promise<TQueue
   }
 
   const pageLimit = formData.get("pageLimit")
-  if (!pageLimit && service === 'DIRECT_UPLOAD_UPDATE') {
+
+  if (!pageLimit) {
     const connection = user.connections.find(conn => conn.id === connectionId)
     if (connection && connection.limitPages) formData.set("pageLimit", connection.limitPages.toString())
   }
diff --git a/fileProcessors/index.ts b/fileProcessors/index.ts
index 529386e..3a3f726 100644
--- a/fileProcessors/index.ts
+++ b/fileProcessors/index.ts
@@ -104,6 +104,23 @@ export const connectionProcessFiles = async ({ connectionId, service, pageLimit,
       points: file.chunksIds,
     })
   }
+  // check the limits 
+  const currentPagesCount = connection?.files.reduce((sum, f) => f.totalPages + sum, 0) ?? 0
+  if (pageLimit && pageLimit < currentPagesCount) {
+    await databaseDrizzle
+      .update(connections)
+      .set({ isSyncing: false })
+      .where(eq(connections.id, connectionId))
+
+    await publishProgress({
+      connectionId: connectionId,
+      processedFile: connection.files.length,
+      processedPage: currentPagesCount,
+      lastAsync: new Date(),
+      status: 'FINISHED',
+    })
+    return;
+  }
   return processFiles(filesContent, service, connectionId, pageLimit, fileLimit, 0, 0)
 }
 
diff --git a/mocks/handlers.ts b/mocks/handlers.ts
index f8dd904..ed7d9f6 100644
--- a/mocks/handlers.ts
+++ b/mocks/handlers.ts
@@ -15,13 +15,13 @@ export const handlers = [
     // resolve it under cypress/fixtures
     const fullPath = path.resolve(process.cwd(), 'cypress', 'fixtures', filename)
     const buffer = fs.readFileSync(fullPath)
-
-    return HttpResponse.arrayBuffer(buffer, {
+   const uint8 = new Uint8Array(buffer)  
+    const arrayBuffer = uint8.buffer
+    return HttpResponse.arrayBuffer(arrayBuffer, {
       headers: {
         'Content-Type': 'application/pdf'
       },
     })
-
   }),
   http.post("https://api.dropboxapi.com/2/files/list_folder", async (res) => {
     const body = (await res.request.json()) as DropboxFile