Add asset store support to AI agent

This commit is contained in:
Florian Rival
2025-05-21 23:04:21 +02:00
parent a7b27b4d2d
commit babf0153a1
11 changed files with 331 additions and 86 deletions

View File

@@ -44,6 +44,8 @@ import UrlStorageProvider from '../ProjectsStorage/UrlStorageProvider';
import { type FileMetadata, type StorageProvider } from '../ProjectsStorage';
import { useEnsureExtensionInstalled } from './UseEnsureExtensionInstalled';
import { useGenerateEvents } from './UseGenerateEvents';
import { useSearchAndInstallAsset } from './UseSearchAndInstallAsset';
import { type ResourceManagementProps } from '../ResourcesList/ResourceSource';
const useEditorFunctionCallResultsPerRequest = () => {
const [
@@ -105,13 +107,15 @@ const useEditorFunctionCallResultsPerRequest = () => {
const useProcessFunctionCalls = ({
i18n,
project,
resourceManagementProps,
selectedAiRequest,
onSendEditorFunctionCallResults,
getEditorFunctionCallResults,
addEditorFunctionCallResults,
}: {|
i18n: I18nType,
project: ?gdProject,
project: gdProject | null,
resourceManagementProps: ResourceManagementProps,
selectedAiRequest: ?AiRequest,
onSendEditorFunctionCallResults: () => Promise<void>,
getEditorFunctionCallResults: string => Array<EditorFunctionCallResult> | null,
@@ -124,7 +128,11 @@ const useProcessFunctionCalls = ({
project,
i18n,
});
const { launchEventsGeneration } = useGenerateEvents({ project });
const { searchAndInstallAsset } = useSearchAndInstallAsset({
project,
resourceManagementProps,
});
const { generateEvents } = useGenerateEvents({ project });
const triggerSendEditorFunctionCallResults = useTriggerAtNextRender(
onSendEditorFunctionCallResults
@@ -179,15 +187,14 @@ const useProcessFunctionCalls = ({
call_id: functionCall.call_id,
})),
ignore: !!options && !!options.ignore,
launchEventsGeneration: async options => {
return await launchEventsGeneration({
generateEvents: async options => {
return await generateEvents({
...options,
relatedAiRequestId: selectedAiRequest.id,
});
},
onEnsureExtensionInstalled: async ({ extensionName }) => {
await ensureExtensionInstalled(extensionName);
},
ensureExtensionInstalled,
searchAndInstallAsset,
});
addEditorFunctionCallResults(
@@ -204,7 +211,8 @@ const useProcessFunctionCalls = ({
selectedAiRequest,
addEditorFunctionCallResults,
ensureExtensionInstalled,
launchEventsGeneration,
searchAndInstallAsset,
generateEvents,
triggerSendEditorFunctionCallResults,
]
);
@@ -425,7 +433,8 @@ const styles = {
type Props = {|
isActive: boolean,
project: ?gdProject,
project: gdProject | null,
resourceManagementProps: ResourceManagementProps,
fileMetadata: ?FileMetadata,
storageProvider: ?StorageProvider,
setToolbar: (?React.Node) => void,
@@ -461,6 +470,7 @@ export const AskAiEditor = React.memo<Props>(
isActive,
setToolbar,
project,
resourceManagementProps,
fileMetadata,
storageProvider,
i18n,
@@ -837,6 +847,7 @@ export const AskAiEditor = React.memo<Props>(
onProcessFunctionCalls,
} = useProcessFunctionCalls({
project,
resourceManagementProps,
selectedAiRequest,
onSendEditorFunctionCallResults,
getEditorFunctionCallResults,
@@ -919,7 +930,8 @@ export const renderAskAiEditorContainer = (
<AskAiEditor
ref={props.ref}
i18n={i18n}
project={props.project}
project={props.project || null}
resourceManagementProps={props.resourceManagementProps}
fileMetadata={props.fileMetadata}
storageProvider={props.storageProvider}
setToolbar={props.setToolbar}

View File

@@ -1,50 +1,54 @@
// @flow
import * as React from "react";
import * as React from 'react';
import { type I18n as I18nType } from '@lingui/core';
import { ExtensionStoreContext } from "../AssetStore/ExtensionStore/ExtensionStoreContext";
import EventsFunctionsExtensionsContext from "../EventsFunctionsExtensionsLoader/EventsFunctionsExtensionsContext";
import { installExtension } from "../AssetStore/ExtensionStore/InstallExtension";
import { ExtensionStoreContext } from '../AssetStore/ExtensionStore/ExtensionStoreContext';
import EventsFunctionsExtensionsContext from '../EventsFunctionsExtensionsLoader/EventsFunctionsExtensionsContext';
import { installExtension } from '../AssetStore/ExtensionStore/InstallExtension';
type EnsureExtensionInstalledOptions = {|
extensionName: string,
|};
export const useEnsureExtensionInstalled = ({
project,
i18n,
}: {|
project: ?gdProject,
i18n: I18nType,
|}) => {
const { translatedExtensionShortHeadersByName } = React.useContext(
ExtensionStoreContext
);
const eventsFunctionsExtensionsState = React.useContext(
EventsFunctionsExtensionsContext
);
project,
i18n,
}: {|
project: ?gdProject,
i18n: I18nType,
|}) => {
const { translatedExtensionShortHeadersByName } = React.useContext(
ExtensionStoreContext
);
const eventsFunctionsExtensionsState = React.useContext(
EventsFunctionsExtensionsContext
);
return {
ensureExtensionInstalled: React.useCallback(
async (extensionName: string) => {
if (!project) return;
if (project.getCurrentPlatform().isExtensionLoaded(extensionName))
return;
return {
ensureExtensionInstalled: React.useCallback(
async ({ extensionName }: EnsureExtensionInstalledOptions) => {
if (!project) return;
if (project.getCurrentPlatform().isExtensionLoaded(extensionName))
return;
const extensionShortHeader =
translatedExtensionShortHeadersByName[extensionName];
if (!extensionShortHeader) {
throw new Error("Can't find extension with the required name.");
}
const extensionShortHeader =
translatedExtensionShortHeadersByName[extensionName];
if (!extensionShortHeader) {
throw new Error("Can't find extension with the required name.");
}
await installExtension(
i18n,
project,
eventsFunctionsExtensionsState,
extensionShortHeader
);
},
[
eventsFunctionsExtensionsState,
await installExtension(
i18n,
project,
translatedExtensionShortHeadersByName,
]
),
};
eventsFunctionsExtensionsState,
extensionShortHeader
);
},
[
eventsFunctionsExtensionsState,
i18n,
project,
translatedExtensionShortHeadersByName,
]
),
};
};

View File

@@ -16,7 +16,7 @@ export const useGenerateEvents = ({ project }: {| project: ?gdProject |}) => {
AuthenticatedUserContext
);
const launchEventsGeneration = React.useCallback(
const generateEvents = React.useCallback(
async ({
sceneName,
eventsDescription,
@@ -94,5 +94,5 @@ export const useGenerateEvents = ({ project }: {| project: ?gdProject |}) => {
[getAuthorizationHeader, project, profile]
);
return { launchEventsGeneration };
return { generateEvents };
};

View File

@@ -0,0 +1,82 @@
// @flow
import * as React from 'react';
import {
type AssetSearchAndInstallOptions,
type AssetSearchAndInstallResult,
} from '../EditorFunctions';
import AuthenticatedUserContext from '../Profile/AuthenticatedUserContext';
import {
createAssetSearch,
type AssetSearch,
} from '../Utils/GDevelopServices/Generation';
import { retryIfFailed } from '../Utils/RetryIfFailed';
import { useInstallAsset } from '../AssetStore/NewObjectDialog';
import { type ResourceManagementProps } from '../ResourcesList/ResourceSource';
export const useSearchAndInstallAsset = ({
project,
resourceManagementProps,
}: {|
project: gdProject | null,
resourceManagementProps: ResourceManagementProps,
|}) => {
const { profile, getAuthorizationHeader } = React.useContext(
AuthenticatedUserContext
);
const installAsset = useInstallAsset({
project,
resourceManagementProps,
});
return {
searchAndInstallAsset: React.useCallback(
async ({
scene,
objectName,
...assetSearchOptions
}: AssetSearchAndInstallOptions): Promise<AssetSearchAndInstallResult> => {
if (!profile) throw new Error('User should be authenticated.');
const assetSearch: AssetSearch = await retryIfFailed({ times: 2 }, () =>
createAssetSearch(getAuthorizationHeader, {
userId: profile.id,
...assetSearchOptions,
})
);
if (!assetSearch.results || assetSearch.results.length === 0) {
return {
status: 'nothing-found',
message: 'No assets found.',
createdObjects: [],
};
}
// In the future, we could ask the user to select the asset they want to use.
// For now, we just return the first asset.
const chosenResult = assetSearch.results[0];
if (!chosenResult) throw new Error('No asset found.');
const installOutput = await installAsset({
assetShortHeader: chosenResult.asset,
objectsContainer: scene.getObjects(),
requestedObjectName: objectName,
});
if (!installOutput) {
return {
status: 'error',
message: 'Asset found but failed to install asset.',
createdObjects: [],
};
}
return {
status: 'asset-installed',
message: 'Asset installed successfully.',
createdObjects: installOutput.createdObjects,
};
},
[installAsset, profile, getAuthorizationHeader]
),
};
};

View File

@@ -50,7 +50,6 @@ function AssetSwappingDialog({
] = React.useState<boolean>(false);
const installAsset = useInstallAsset({
project,
objectsContainer,
resourceManagementProps,
});
const { showAlert } = useAlertDialog();
@@ -73,7 +72,10 @@ function AssetSwappingDialog({
setIsAssetBeingInstalled(true);
try {
const installAssetOutput = await installAsset(openedAssetShortHeader);
const installAssetOutput = await installAsset({
assetShortHeader: openedAssetShortHeader,
objectsContainer,
});
if (!installAssetOutput) {
throw new Error('Failed to install asset');
}

View File

@@ -153,6 +153,7 @@ export type InstallAssetArgs = {|
project: gdProject,
objectsContainer: gdObjectsContainer,
targetObjectFolderOrObject?: ?gdObjectFolderOrObject,
requestedObjectName?: string,
|};
const findVariant = (
@@ -177,6 +178,7 @@ export const addAssetToProject = async ({
project,
objectsContainer,
targetObjectFolderOrObject,
requestedObjectName,
}: InstallAssetArgs): Promise<InstallAssetOutput> => {
const objectNewNames = {};
const resourceNewNames = {};
@@ -270,7 +272,9 @@ export const addAssetToProject = async ({
}
// Insert the object
const originalName = sanitizeObjectName(objectAsset.object.name);
const originalName = sanitizeObjectName(
requestedObjectName || objectAsset.object.name
);
const newName = newNameGenerator(originalName, name =>
objectsContainer.hasObjectNamed(name)
);

View File

@@ -123,12 +123,10 @@ export const useFetchAssets = () => {
export const useInstallAsset = ({
project,
objectsContainer,
targetObjectFolderOrObjectWithContext,
resourceManagementProps,
}: {|
project: gdProject,
objectsContainer: gdObjectsContainer,
project: gdProject | null,
targetObjectFolderOrObjectWithContext?: ?ObjectFolderOrObjectWithContext,
resourceManagementProps: ResourceManagementProps,
|}) => {
@@ -147,9 +145,18 @@ export const useInstallAsset = ({
resourceManagementProps.canInstallPrivateAsset
);
return async (
assetShortHeader: AssetShortHeader
): Promise<InstallAssetOutput | null> => {
return async ({
assetShortHeader,
objectsContainer,
requestedObjectName,
}: {|
assetShortHeader: AssetShortHeader,
objectsContainer: gdObjectsContainer,
requestedObjectName?: string,
|}): Promise<InstallAssetOutput | null> => {
if (!project) {
return null;
}
try {
if (await showProjectNeedToBeSaved(assetShortHeader)) {
return null;
@@ -190,6 +197,7 @@ export const useInstallAsset = ({
asset,
project,
objectsContainer,
requestedObjectName,
targetObjectFolderOrObject:
targetObjectFolderOrObjectWithContext &&
!targetObjectFolderOrObjectWithContext.global
@@ -200,6 +208,7 @@ export const useInstallAsset = ({
asset,
project,
objectsContainer,
requestedObjectName,
targetObjectFolderOrObject:
targetObjectFolderOrObjectWithContext &&
!targetObjectFolderOrObjectWithContext.global
@@ -315,7 +324,6 @@ function NewObjectDialog({
const showExtensionUpdateConfirmation = useExtensionUpdateAlertDialog();
const installAsset = useInstallAsset({
project,
objectsContainer,
resourceManagementProps,
targetObjectFolderOrObjectWithContext,
});
@@ -325,13 +333,16 @@ function NewObjectDialog({
if (!assetShortHeader) return false;
setIsAssetBeingInstalled(true);
const installAssetOutput = await installAsset(assetShortHeader);
const installAssetOutput = await installAsset({
assetShortHeader,
objectsContainer,
});
setIsAssetBeingInstalled(false);
if (installAssetOutput)
onObjectsAddedFromAssets(installAssetOutput.createdObjects);
return !!installAssetOutput;
},
[installAsset, onObjectsAddedFromAssets]
[installAsset, onObjectsAddedFromAssets, objectsContainer]
);
const onInstallEmptyCustomObject = React.useCallback(

View File

@@ -114,6 +114,7 @@ const PrivateAssetsAuthorizationProvider = ({ children }: Props) => {
asset,
project,
objectsContainer,
requestedObjectName,
targetObjectFolderOrObject,
}: InstallAssetArgs): Promise<?InstallAssetOutput> => {
if (!profile) {
@@ -134,6 +135,7 @@ const PrivateAssetsAuthorizationProvider = ({ children }: Props) => {
asset: assetWithAuthorizedResourceUrls,
project,
objectsContainer,
requestedObjectName,
targetObjectFolderOrObject,
});
};

View File

@@ -5,6 +5,8 @@ import {
type EditorFunction,
type EditorFunctionCall,
type EventsGenerationOptions,
type AssetSearchAndInstallOptions,
type AssetSearchAndInstallResult,
} from '.';
export type EditorFunctionCallResult =
@@ -39,20 +41,24 @@ export type ProcessEditorFunctionCallsOptions = {|
project: gdProject,
functionCalls: Array<EditorFunctionCall>,
ignore: boolean,
launchEventsGeneration: (
generateEvents: (
options: EventsGenerationOptions
) => Promise<EventsGenerationResult>,
onEnsureExtensionInstalled: (options: {
ensureExtensionInstalled: (options: {|
extensionName: string,
}) => Promise<void>,
|}) => Promise<void>,
searchAndInstallAsset: (
options: AssetSearchAndInstallOptions
) => Promise<AssetSearchAndInstallResult>,
|};
export const processEditorFunctionCalls = async ({
functionCalls,
project,
launchEventsGeneration,
generateEvents,
ignore,
onEnsureExtensionInstalled,
ensureExtensionInstalled,
searchAndInstallAsset,
}: ProcessEditorFunctionCallsOptions): Promise<
Array<EditorFunctionCallResult>
> => {
@@ -129,8 +135,9 @@ export const processEditorFunctionCalls = async ({
{
project,
args,
launchEventsGeneration,
onEnsureExtensionInstalled,
generateEvents,
ensureExtensionInstalled,
searchAndInstallAsset,
}
);
const { success, ...output } = result;

View File

@@ -66,6 +66,21 @@ export type EventsGenerationOptions = {|
placementHint: string,
|};
export type AssetSearchAndInstallResult = {|
status: 'asset-installed' | 'nothing-found' | 'error',
message: string,
createdObjects: Array<gdObject>,
|};
export type AssetSearchAndInstallOptions = {|
scene: gdLayout,
objectName: string,
objectType: string,
searchTerms: string,
description: string,
twoDimensionalViewKind: string,
|};
export type EditorCallbacks = {|
onOpenLayout: (sceneName: string) => void,
onOpenEvents: (sceneName: string) => void,
@@ -83,12 +98,15 @@ export type EditorFunction = {|
launchFunction: (options: {|
project: gdProject,
args: any,
launchEventsGeneration: (
generateEvents: (
options: EventsGenerationOptions
) => Promise<EventsGenerationResult>,
onEnsureExtensionInstalled: (options: {
ensureExtensionInstalled: (options: {|
extensionName: string,
}) => Promise<void>,
|}) => Promise<void>,
searchAndInstallAsset: (
options: AssetSearchAndInstallOptions
) => Promise<AssetSearchAndInstallResult>,
|}) => Promise<EditorFunctionGenericOutput>,
|};
@@ -161,10 +179,27 @@ const createObject: EditorFunction = {
</Trans>
);
},
launchFunction: async ({ project, args, onEnsureExtensionInstalled }) => {
launchFunction: async ({
project,
args,
ensureExtensionInstalled,
searchAndInstallAsset,
}) => {
const scene_name = extractRequiredString(args, 'scene_name');
const object_type = extractRequiredString(args, 'object_type');
const object_name = extractRequiredString(args, 'object_name');
const description = SafeExtractor.extractStringProperty(
args,
'description'
);
const search_terms = SafeExtractor.extractStringProperty(
args,
'search_terms'
);
const two_dimensional_view_kind = SafeExtractor.extractStringProperty(
args,
'two_dimensional_view_kind'
);
if (!project.hasLayoutNamed(scene_name)) {
return makeGenericFailure(`Scene not found: "${scene_name}".`);
@@ -186,10 +221,44 @@ const createObject: EditorFunction = {
);
}
// First try to search and install an object from the asset store.
try {
const { status, message, createdObjects } = await searchAndInstallAsset({
scene: layout,
objectName: object_name,
objectType: object_type,
searchTerms: search_terms,
description,
twoDimensionalViewKind: two_dimensional_view_kind,
});
if (status === 'error') {
return makeGenericFailure(
`Unable to search and install object (${message}).`
);
} else if (status === 'asset-installed') {
return makeGenericSuccess(
`Created (from the asset store) ${createdObjects
.map(object => `object "${object_name}" of type "${object_type}"`)
.join(', ')} in scene "${scene_name}".`
);
} else {
// No asset found - we'll create an object from scratch.
}
} catch (error) {
return makeGenericFailure(
`An unexpected error happened while search and installing objects (${
error.message
}).`
);
}
// Create an object from scratch:
// Ensure the extension for this object type is installed.
if (object_type.includes('::')) {
const extensionName = object_type.split('::')[0];
try {
await onEnsureExtensionInstalled({ extensionName });
await ensureExtensionInstalled({ extensionName });
} catch (error) {
console.error(
`Could not get extension "${extensionName}" installed:`,
@@ -201,6 +270,7 @@ const createObject: EditorFunction = {
}
}
// Ensure the object type is valid.
const objectMetadata = gd.MetadataProvider.getObjectMetadata(
project.getCurrentPlatform(),
object_type
@@ -211,17 +281,14 @@ const createObject: EditorFunction = {
);
}
// Create the object based on the type
objectsContainer.insertNewObject(
project,
object_type,
object_name,
objectsContainer.getObjectsCount()
);
// TODO: send back the properties of the object?
return makeGenericSuccess(
`Created object "${object_name}" of type "${object_type}" in scene "${scene_name}".`
`Created a new object (from scratch) called "${object_name}" of type "${object_type}" in scene "${scene_name}".`
);
},
};
@@ -416,7 +483,7 @@ const addBehavior: EditorFunction = {
return makeText(behaviorMetadata.getFullName());
},
launchFunction: async ({ project, args, onEnsureExtensionInstalled }) => {
launchFunction: async ({ project, args, ensureExtensionInstalled }) => {
const scene_name = extractRequiredString(args, 'scene_name');
const object_name = extractRequiredString(args, 'object_name');
const behavior_type = extractRequiredString(args, 'behavior_type');
@@ -444,7 +511,7 @@ const addBehavior: EditorFunction = {
if (behavior_type.includes('::')) {
const extensionName = behavior_type.split('::')[0];
try {
await onEnsureExtensionInstalled({ extensionName });
await ensureExtensionInstalled({ extensionName });
} catch (error) {
console.error(
`Could not get extension "${extensionName}" installed:`,
@@ -1015,8 +1082,8 @@ const addSceneEvents: EditorFunction = {
launchFunction: async ({
project,
args,
launchEventsGeneration,
onEnsureExtensionInstalled,
generateEvents,
ensureExtensionInstalled,
}) => {
const sceneName = extractRequiredString(args, 'scene_name');
const eventsDescription = extractRequiredString(args, 'events_description');
@@ -1045,7 +1112,7 @@ const addSceneEvents: EditorFunction = {
});
try {
const eventsGenerationResult = await launchEventsGeneration({
const eventsGenerationResult = await generateEvents({
sceneName,
eventsDescription,
extensionNamesList,
@@ -1089,7 +1156,7 @@ const addSceneEvents: EditorFunction = {
for (const change of changes) {
for (const extensionName of change.extensionNames || []) {
await onEnsureExtensionInstalled({ extensionName });
await ensureExtensionInstalled({ extensionName });
}
}

View File

@@ -128,6 +128,23 @@ export type AiGeneratedEvent = {
stats: AiGeneratedEventStats | null,
};
export type AssetSearch = {
id: string,
userId: string,
createdAt: string,
query: {
searchTerms: string[],
objectType: string,
description: string | null,
twoDimensionalViewKind: string | null,
},
status: 'completed' | 'failed',
results: Array<{
score: number,
asset: any,
}> | null,
};
export const getGeneratedProject = async (
getAuthorizationHeader: () => Promise<string>,
{
@@ -488,3 +505,40 @@ export const getAiGeneratedEvent = async (
);
return response.data;
};
export const createAssetSearch = async (
getAuthorizationHeader: () => Promise<string>,
{
userId,
searchTerms,
description,
objectType,
twoDimensionalViewKind,
}: {|
userId: string,
searchTerms: string,
description: string,
objectType: string,
twoDimensionalViewKind: string,
|}
): Promise<AssetSearch> => {
const authorizationHeader = await getAuthorizationHeader();
const response = await axios.post(
`${GDevelopGenerationApi.baseUrl}/asset-search`,
{
searchTerms,
description,
objectType,
twoDimensionalViewKind,
},
{
params: {
userId,
},
headers: {
Authorization: authorizationHeader,
},
}
);
return response.data;
};