From 770e4d106fe8bf33f20426fcc9feb74d4f89ac31 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Tue, 5 May 2026 10:43:52 +0200 Subject: [PATCH 1/2] Use catalog components instead of locations Signed-off-by: Dominika Zemanovicova --- .../__fixtures__/testUtils.ts | 34 ++-- .../src/catalog/catalogHttpClient.ts | 36 ++++ .../repository/repositories-gitlab.test.ts | 144 ++++++++-------- .../handlers/repository/repositories.test.ts | 157 +++++++++--------- .../handlers/repository/repositories.ts | 11 +- 5 files changed, 197 insertions(+), 185 deletions(-) diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/testUtils.ts b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/testUtils.ts index a10f2136c8..ab317e9df7 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/testUtils.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/__fixtures__/testUtils.ts @@ -18,9 +18,16 @@ import { BackendFeature, createServiceFactory, } from '@backstage/backend-plugin-api'; -import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; -import type { CatalogClient } from '@backstage/catalog-client'; +import { + mockServices, + startTestBackend, + type ServiceMock, +} from '@backstage/backend-test-utils'; import { catalogServiceRef } from '@backstage/plugin-catalog-node/alpha'; +import { + catalogServiceMock, + type CatalogServiceMock, +} from '@backstage/plugin-catalog-node/testUtils'; import { AuthorizeResult } from '@backstage/plugin-permission-common'; import { rest } from 'msw'; @@ -170,7 +177,7 @@ export function addHandlersForGHTokenAppErrors(server: SetupServer) { } export async function startBackendServer( - mockCatalogClient: CatalogClient, + mockCatalogClient: ServiceMock, authorizeResult?: AuthorizeResult.DENY | AuthorizeResult.ALLOW, config?: any, db?: any, @@ -201,7 +208,7 @@ export async function startBackendServer( export function setupTest() { let server: SetupServer; - let mockCatalogClient: CatalogClient; + let mockCatalogClient: ServiceMock; beforeAll(() => { server = setupServer(...DEFAULT_TEST_HANDLERS); @@ -223,24 +230,7 @@ export function setupTest() { afterAll(() => server.close()); beforeEach(() => { - // TODO(rm3l): Use 'catalogServiceMock' from '@backstage/plugin-catalog-node/testUtils' - // once '@backstage/plugin-catalog-node' is upgraded - mockCatalogClient = { - getEntities: jest.fn(), - getEntitiesByRefs: jest.fn(), - queryEntities: jest.fn(), - getEntityAncestors: jest.fn(), - getEntityByRef: jest.fn(), - removeEntityByUid: jest.fn(), - refreshEntity: jest.fn(), - getEntityFacets: jest.fn(), - getLocationById: jest.fn(), - getLocationByRef: jest.fn(), - addLocation: jest.fn(), - removeLocationById: jest.fn(), - getLocationByEntity: jest.fn(), - validateEntity: jest.fn(), - } as unknown as CatalogClient; + mockCatalogClient = catalogServiceMock.mock(); }); afterEach(() => { diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts index a7533ac8e8..639f1b3700 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/catalog/catalogHttpClient.ts @@ -102,6 +102,42 @@ export class CatalogHttpClient { }; } + async listCatalogComponentManagedByUrlLocations(): Promise> { + const result = await this.catalogApi.queryEntities( + { + fields: ['metadata.annotations'], + query: { + $all: [ + { kind: 'Component' }, + { + 'metadata.annotations.backstage.io/managed-by-location': { + $hasPrefix: 'url:', + }, + }, + ], + }, + // We need all imported component targets to reliably exclude already-imported repositories. + // That's why we are retrieving this hard-coded high number of Components. + limit: 9999, + }, + { + token: await getTokenForPlugin(this.auth, 'catalog'), + }, + ); + + const entities = result?.items ?? []; + return new Set( + entities + .map( + entity => + entity.metadata.annotations?.[ + 'backstage.io/managed-by-location' + ]?.slice(4), // remove 'url:' prefix + ) + .filter((repoUrl): repoUrl is string => !!repoUrl), + ); + } + async listCatalogUrlLocationsById( search?: string, pageNumber?: number, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts index ba824686fb..0a04c7e80e 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories-gitlab.test.ts @@ -19,10 +19,7 @@ import { AuthorizeResult } from '@backstage/plugin-permission-common'; import { rest } from 'msw'; import request from 'supertest'; -import { - CATALOG_API_LOCATIONS_LOCAL_ADDR, - LOCAL_ADDR, -} from '../../../../__fixtures__/handlers'; +import { LOCAL_ADDR } from '../../../../__fixtures__/handlers'; import { setupTest, startBackendServer, @@ -210,25 +207,24 @@ describe('repositories', () => { }); it('returns filtered (not yet imported) repos when some repos are already imported', async () => { - const { server, mockCatalogClient } = useTestData(); - - server.use( - rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => - res( - ctx.status(200), - ctx.json([ - { - data: { - id: 'imported-funtimes', - target: - 'http://localhost:8765/saltypig1/funtimes/blob/main/catalog-info.yaml', - type: 'url', - }, + const { mockCatalogClient } = useTestData(); + mockCatalogClient.queryEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'funtimes', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/saltypig1/funtimes/blob/main/catalog-info.yaml', }, - ]), - ), - ), - ); + }, + }, + ], + totalItems: 1, + pageInfo: {}, + }); const backendServer = await startBackendServer( mockCatalogClient, @@ -269,41 +265,46 @@ describe('repositories', () => { }); it('returns empty array when all repos are already imported', async () => { - const { server, mockCatalogClient } = useTestData(); - - server.use( - rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => - res( - ctx.status(200), - ctx.json([ - { - data: { - id: 'imported-dolbear', - target: - 'http://localhost:8765/saltypig1/dolbear/blob/main/catalog-info.yaml', - type: 'url', - }, + const { mockCatalogClient } = useTestData(); + mockCatalogClient.queryEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'dolbear', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/saltypig1/dolbear/blob/main/catalog-info.yaml', }, - { - data: { - id: 'imported-funtimes', - target: - 'http://localhost:8765/saltypig1/funtimes/blob/main/catalog-info.yaml', - type: 'url', - }, + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'funtimes', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/saltypig1/funtimes/blob/main/catalog-info.yaml', }, - { - data: { - id: 'imported-swapi-node', - target: - 'http://localhost:8765/saltypig1/swapi-node/blob/main/catalog-info.yaml', - type: 'url', - }, + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'swapi-node', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/saltypig1/swapi-node/blob/main/catalog-info.yaml', }, - ]), - ), - ), - ); + }, + }, + ], + totalItems: 3, + pageInfo: {}, + }); const backendServer = await startBackendServer( mockCatalogClient, @@ -325,25 +326,24 @@ describe('repositories', () => { }); it('returns all repos even though a non-root catalog location exists', async () => { - const { server, mockCatalogClient } = useTestData(); - - server.use( - rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => - res( - ctx.status(200), - ctx.json([ - { - data: { - id: 'imported-funtimes-nested', - target: - 'http://localhost:8765/saltypig1/funtimes/blob/main/packages/backend/catalog-info.yaml', - type: 'url', - }, + const { mockCatalogClient } = useTestData(); + mockCatalogClient.queryEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'funtimes-nested', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/saltypig1/funtimes/blob/main/packages/backend/catalog-info.yaml', }, - ]), - ), - ), - ); + }, + }, + ], + totalItems: 1, + pageInfo: {}, + }); const backendServer = await startBackendServer( mockCatalogClient, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts index ae5a98252d..ebda051dfc 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.test.ts @@ -19,10 +19,7 @@ import { AuthorizeResult } from '@backstage/plugin-permission-common'; import { rest } from 'msw'; import request from 'supertest'; -import { - CATALOG_API_LOCATIONS_LOCAL_ADDR, - LOCAL_ADDR, -} from '../../../../__fixtures__/handlers'; +import { LOCAL_ADDR } from '../../../../__fixtures__/handlers'; import { addHandlersForGHTokenAppErrors, setupTest, @@ -220,25 +217,35 @@ describe('repositories', () => { }); it('returns filtered (not yet imported) repos when some repos are already imported', async () => { - const { server, mockCatalogClient } = useTestData(); - - server.use( - rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => - res( - ctx.status(200), - ctx.json([ - { - data: { - id: 'imported-hello-world', - target: - 'http://localhost:8765/octocat/Hello-World/blob/master/catalog-info.yaml', - type: 'url', - }, + const { mockCatalogClient } = useTestData(); + mockCatalogClient.queryEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'hello-world', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/octocat/Hello-World/blob/master/catalog-info.yaml', }, - ]), - ), - ), - ); + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'animated-happiness', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/octocat/animated-happiness/blob/master/catalog-info.yaml', + }, + }, + }, + ], + totalItems: 2, + pageInfo: {}, + }); const backendServer = await startBackendServer( mockCatalogClient, @@ -253,15 +260,6 @@ describe('repositories', () => { expect(response.body).toEqual({ errors: [], repositories: [ - { - defaultBranch: 'master', - errors: [], - id: 'octocat/animated-happiness', - lastUpdate: '2011-01-26T19:14:43Z', - name: 'animated-happiness', - organization: 'octocat', - url: 'http://localhost:8765/octocat/animated-happiness', - }, { defaultBranch: 'master', errors: [], @@ -272,46 +270,40 @@ describe('repositories', () => { url: 'http://localhost:8765/my-user/Lorem-Ipsum', }, ], - totalCount: 2, + totalCount: 1, }); }); it('returns empty array when all repos are already imported', async () => { - const { server, mockCatalogClient } = useTestData(); - - server.use( - rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => - res( - ctx.status(200), - ctx.json([ - { - data: { - id: 'imported-animated-happiness', - target: - 'http://localhost:8765/octocat/animated-happiness/blob/master/catalog-info.yaml', - type: 'url', - }, - }, - { - data: { - id: 'imported-hello-world', - target: - 'http://localhost:8765/octocat/Hello-World/blob/master/catalog-info.yaml', - type: 'url', - }, + const { mockCatalogClient } = useTestData(); + mockCatalogClient.queryEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'animated-happiness', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/octocat/animated-happiness/blob/master/catalog-info.yaml', }, - { - data: { - id: 'imported-lorem-ipsum', - target: - 'http://localhost:8765/my-user/Lorem-Ipsum/blob/master/catalog-info.yaml', - type: 'url', - }, + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'lorem-ipsum', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/my-user/Lorem-Ipsum/blob/master/catalog-info.yaml', }, - ]), - ), - ), - ); + }, + }, + ], + totalItems: 2, + pageInfo: {}, + }); const backendServer = await startBackendServer( mockCatalogClient, @@ -331,25 +323,24 @@ describe('repositories', () => { }); it('returns all repos even though a non-root catalog location exists', async () => { - const { server, mockCatalogClient } = useTestData(); - - server.use( - rest.get(CATALOG_API_LOCATIONS_LOCAL_ADDR, (_, res, ctx) => - res( - ctx.status(200), - ctx.json([ - { - data: { - id: 'imported-animated-happiness-sub', - target: - 'http://localhost:8765/octocat/animated-happiness/blob/master/monorepo/nested/path/catalog-info.yaml', - type: 'url', - }, + const { mockCatalogClient } = useTestData(); + mockCatalogClient.queryEntities.mockResolvedValue({ + items: [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'animated-happiness-nested', + annotations: { + 'backstage.io/managed-by-location': + 'url:http://localhost:8765/octocat/animated-happiness/blob/master/monorepo/nested/path/catalog-info.yaml', }, - ]), - ), - ), - ); + }, + }, + ], + totalItems: 1, + pageInfo: {}, + }); const backendServer = await startBackendServer( mockCatalogClient, diff --git a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts index 04da6d49f8..81c26ccbca 100644 --- a/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts +++ b/workspaces/bulk-import/plugins/bulk-import-backend/src/service/handlers/repository/repositories.ts @@ -59,14 +59,10 @@ export async function findAllRepositories( const [alreadyImportedRepositories, allRepositoriesResponse] = await Promise.all([ - deps.catalogHttpClient.listCatalogUrlLocations(), + deps.catalogHttpClient.listCatalogComponentManagedByUrlLocations(), deps.gitApiService.getRepositoriesFromIntegrations(search, userTokens), ]); - const alreadyImportedRepositoriesLocationTargets = new Set( - alreadyImportedRepositories.uniqueCatalogUrlLocations.keys(), - ); - const { repositories: allRepositories, errors } = allRepositoriesResponse; const notImportedYetRepositories = allRepositories.filter(repo => { @@ -76,8 +72,7 @@ export async function findAllRepositories( repo.default_branch, ); - let alreadyImported = - alreadyImportedRepositoriesLocationTargets.has(catalogUrl); + let alreadyImported = alreadyImportedRepositories.has(catalogUrl); if (!alreadyImported) { // Workaround: when a GitHub repository is imported via Backstage, the @@ -87,7 +82,7 @@ export async function findAllRepositories( // format was persisted, the '/tree/' variant is also checked here. // This branch can be removed once catalog locations are consistently // stored using the same '/blob/' format as getCatalogUrl returns. - alreadyImported = alreadyImportedRepositoriesLocationTargets.has( + alreadyImported = alreadyImportedRepositories.has( catalogUrl.replace('/blob/', '/tree/'), ); } From 46d3009543a916a1733823717b8c14b9bc0b7780 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Tue, 5 May 2026 10:58:51 +0200 Subject: [PATCH 2/2] Add changeset Signed-off-by: Dominika Zemanovicova --- workspaces/bulk-import/.changeset/humble-moons-wait.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 workspaces/bulk-import/.changeset/humble-moons-wait.md diff --git a/workspaces/bulk-import/.changeset/humble-moons-wait.md b/workspaces/bulk-import/.changeset/humble-moons-wait.md new file mode 100644 index 0000000000..c532e01d7d --- /dev/null +++ b/workspaces/bulk-import/.changeset/humble-moons-wait.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-bulk-import-backend': patch +--- + +Correctly identify already-imported repositories, ensuring repos with pending import PRs are visible for import.