feat: added dynamic variables creation flow (#7541)

* feat: added dynamic variables creation flow

* feat: added keys and value apis and hooks

* feat: added api and select component changes

* feat: added keys fetching and preview values

* feat: added dynamic variable to variable items

* feat: handled value persistence and tab switches

* feat: added default value and formed a schema for dyn-variables

* feat: added client and server side searches

* feat: corrected the initial load getfieldKey api

* feat: removed fetch on mount restriction

frontend/src/api/dynamicVariables/getFieldKeys.ts

@@ -0,0 +1,34 @@
+import { ApiBaseInstance } from 'api';
+import { ErrorResponse, SuccessResponse } from 'types/api';
+import { FieldKeyResponse } from 'types/api/dynamicVariables/getFieldKeys';
+
+/**
+ * Get field keys for a given signal type
+ * @param signal Type of signal (traces, logs, metrics)
+ * @param name Optional search text
+ */
+export const getFieldKeys = async (
+	signal?: 'traces' | 'logs' | 'metrics',
+	name?: string,
+): Promise<SuccessResponse<FieldKeyResponse> | ErrorResponse> => {
+	const params: Record<string, string> = {};
+
+	if (signal) {
+		params.signal = signal;
+	}
+
+	if (name) {
+		params.name = name;
+	}
+
+	const response = await ApiBaseInstance.get('/fields/keys', { params });
+
+	return {
+		statusCode: 200,
+		error: null,
+		message: response.data.status,
+		payload: response.data.data,
+	};
+};
+
+export default getFieldKeys;

frontend/src/api/dynamicVariables/getFieldValues.ts

@@ -0,0 +1,40 @@
+import { ApiBaseInstance } from 'api';
+import { ErrorResponse, SuccessResponse } from 'types/api';
+import { FieldValueResponse } from 'types/api/dynamicVariables/getFieldValues';
+
+/**
+ * Get field values for a given signal type and field name
+ * @param signal Type of signal (traces, logs, metrics)
+ * @param name Name of the attribute for which values are being fetched
+ * @param value Optional search text
+ */
+export const getFieldValues = async (
+	signal?: 'traces' | 'logs' | 'metrics',
+	name?: string,
+	value?: string,
+): Promise<SuccessResponse<FieldValueResponse> | ErrorResponse> => {
+	const params: Record<string, string> = {};
+
+	if (signal) {
+		params.signal = signal;
+	}
+
+	if (name) {
+		params.name = name;
+	}
+
+	if (value) {
+		params.value = value;
+	}
+
+	const response = await ApiBaseInstance.get('/fields/values', { params });
+
+	return {
+		statusCode: 200,
+		error: null,
+		message: response.data.status,
+		payload: response.data.data,
+	};
+};
+
+export default getFieldValues;

frontend/src/components/NewSelect/types.ts

@@ -24,7 +24,7 @@ export interface CustomSelectProps extends Omit<SelectProps, 'options'> {
 	highlightSearch?: boolean;
 	placement?: 'topLeft' | 'topRight' | 'bottomLeft' | 'bottomRight';
 	popupMatchSelectWidth?: boolean;
-	errorMessage?: string;
+	errorMessage?: string | null;
 	allowClear?: SelectProps['allowClear'];
 	onRetry?: () => void;
 }

frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/DynamicVariable/DynamicVariable.styles.scss

@@ -0,0 +1,42 @@
+.dynamic-variable-container {
+	display: grid;
+	grid-template-columns: 1fr 32px 200px;
+	gap: 32px;
+	align-items: center;
+	width: 100%;
+	margin: 24px 0;
+
+	.ant-select {
+		.ant-select-selector {
+			border-radius: 4px;
+			border: 1px solid var(--bg-slate-400);
+		}
+	}
+
+	.ant-input {
+		border-radius: 4px;
+		border: 1px solid var(--bg-slate-400);
+		max-width: 300px;
+	}
+
+	.dynamic-variable-from-text {
+		font-family: 'Space Mono';
+		font-size: 13px;
+		font-weight: 500;
+		white-space: nowrap;
+	}
+}
+
+.lightMode {
+	.dynamic-variable-container {
+		.ant-select {
+			.ant-select-selector {
+				border: 1px solid var(--bg-vanilla-300);
+			}
+		}
+
+		.ant-input {
+			border: 1px solid var(--bg-vanilla-300);
+		}
+	}
+}

frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/DynamicVariable/DynamicVariable.tsx

@@ -0,0 +1,175 @@
+import './DynamicVariable.styles.scss';
+
+import { Select, Typography } from 'antd';
+import CustomSelect from 'components/NewSelect/CustomSelect';
+import { DEBOUNCE_DELAY } from 'constants/queryBuilderFilterConfig';
+import { useGetFieldKeys } from 'hooks/dynamicVariables/useGetFieldKeys';
+import useDebounce from 'hooks/useDebounce';
+import {
+	Dispatch,
+	SetStateAction,
+	useCallback,
+	useEffect,
+	useMemo,
+	useState,
+} from 'react';
+import { FieldKey } from 'types/api/dynamicVariables/getFieldKeys';
+
+enum AttributeSource {
+	ALL_SOURCES = 'All Sources',
+	LOGS = 'Logs',
+	METRICS = 'Metrics',
+	TRACES = 'Traces',
+}
+
+function DynamicVariable({
+	setDynamicVariablesSelectedValue,
+	dynamicVariablesSelectedValue,
+}: {
+	setDynamicVariablesSelectedValue: Dispatch<
+		SetStateAction<
+			| {
+					name: string;
+					value: string;
+			  }
+			| undefined
+		>
+	>;
+	dynamicVariablesSelectedValue:
+		| {
+				name: string;
+				value: string;
+		  }
+		| undefined;
+}): JSX.Element {
+	const sources = [
+		AttributeSource.ALL_SOURCES,
+		AttributeSource.LOGS,
+		AttributeSource.TRACES,
+		AttributeSource.METRICS,
+	];
+
+	const [attributeSource, setAttributeSource] = useState<AttributeSource>();
+	const [attributes, setAttributes] = useState<Record<string, FieldKey[]>>({});
+	const [selectedAttribute, setSelectedAttribute] = useState<string>();
+	const [apiSearchText, setApiSearchText] = useState<string>('');
+
+	const debouncedApiSearchText = useDebounce(apiSearchText, DEBOUNCE_DELAY);
+
+	const [filteredAttributes, setFilteredAttributes] = useState<
+		Record<string, FieldKey[]>
+	>({});
+
+	useEffect(() => {
+		if (dynamicVariablesSelectedValue?.name) {
+			setSelectedAttribute(dynamicVariablesSelectedValue.name);
+		}
+
+		if (dynamicVariablesSelectedValue?.value) {
+			setAttributeSource(dynamicVariablesSelectedValue.value as AttributeSource);
+		}
+	}, [
+		dynamicVariablesSelectedValue?.name,
+		dynamicVariablesSelectedValue?.value,
+	]);
+
+	const { data, error, isLoading, refetch } = useGetFieldKeys({
+		signal:
+			attributeSource === AttributeSource.ALL_SOURCES
+				? undefined
+				: (attributeSource?.toLowerCase() as 'traces' | 'logs' | 'metrics'),
+		name: debouncedApiSearchText,
+	});
+
+	const isComplete = useMemo(() => data?.payload?.complete === true, [data]);
+
+	useEffect(() => {
+		if (data) {
+			const newAttributes = data.payload?.keys ?? {};
+			setAttributes(newAttributes);
+			setFilteredAttributes(newAttributes);
+		}
+	}, [data]);
+
+	// refetch when attributeSource changes
+	useEffect(() => {
+		if (attributeSource) {
+			refetch();
+		}
+	}, [attributeSource, refetch, debouncedApiSearchText]);
+
+	// Handle search based on whether we have complete data or not
+	const handleSearch = useCallback(
+		(text: string) => {
+			if (isComplete) {
+				// If complete is true, do client-side filtering
+				if (!text) {
+					setFilteredAttributes(attributes);
+					return;
+				}
+
+				const filtered: Record<string, FieldKey[]> = {};
+				Object.keys(attributes).forEach((key) => {
+					if (key.toLowerCase().includes(text.toLowerCase())) {
+						filtered[key] = attributes[key];
+					}
+				});
+				setFilteredAttributes(filtered);
+			} else {
+				// If complete is false, debounce the API call
+				setApiSearchText(text);
+			}
+		},
+		[attributes, isComplete],
+	);
+
+	// update setDynamicVariablesSelectedValue with debounce when attribute and source is selected
+	useEffect(() => {
+		if (selectedAttribute || attributeSource) {
+			setDynamicVariablesSelectedValue({
+				name: selectedAttribute || dynamicVariablesSelectedValue?.name || '',
+				value:
+					attributeSource ||
+					dynamicVariablesSelectedValue?.value ||
+					AttributeSource.ALL_SOURCES,
+			});
+		}
+	}, [
+		selectedAttribute,
+		attributeSource,
+		setDynamicVariablesSelectedValue,
+		dynamicVariablesSelectedValue?.name,
+		dynamicVariablesSelectedValue?.value,
+	]);
+
+	return (
+		<div className="dynamic-variable-container">
+			<CustomSelect
+				placeholder="Select an Attribute"
+				options={Object.keys(filteredAttributes).map((key) => ({
+					label: key,
+					value: key,
+				}))}
+				loading={isLoading}
+				status={error ? 'error' : undefined}
+				onChange={(value): void => {
+					setSelectedAttribute(value);
+				}}
+				showSearch
+				errorMessage={error as any}
+				value={selectedAttribute || dynamicVariablesSelectedValue?.name}
+				onSearch={handleSearch}
+			/>
+			<Typography className="dynamic-variable-from-text">from</Typography>
+			<Select
+				placeholder="Source"
+				defaultValue={AttributeSource.ALL_SOURCES}
+				options={sources.map((source) => ({ label: source, value: source }))}
+				onChange={(value): void => setAttributeSource(value as AttributeSource)}
+				value={attributeSource || dynamicVariablesSelectedValue?.value}
+			/>
+		</div>
+	);
+}
+
+export default DynamicVariable;

frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.styles.scss

@@ -100,7 +100,6 @@
 
 			.variable-type-btn-group {
 				display: flex;
-				width: 342px;
 				height: 32px;
 				flex-shrink: 0;
 				border-radius: 2px;
@@ -199,6 +198,11 @@
 			}
 		}
 
+		.default-value-section {
+			display: grid;
+			grid-template-columns: max-content 1fr;
+		}
+
 		.variable-textbox-section {
 			justify-content: space-between;
 			margin-bottom: 0;

frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx

@@ -6,7 +6,9 @@ import { Button, Collapse, Input, Select, Switch, Tag, Typography } from 'antd';
 import dashboardVariablesQuery from 'api/dashboard/variables/dashboardVariablesQuery';
 import cx from 'classnames';
 import Editor from 'components/Editor';
+import { CustomSelect } from 'components/NewSelect';
 import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
+import { useGetFieldValues } from 'hooks/dynamicVariables/useGetFieldValues';
 import { commaValuesParser } from 'lib/dashbaordVariables/customCommaValuesParser';
 import sortValues from 'lib/dashbaordVariables/sortVariableValues';
 import { map } from 'lodash-es';
@@ -16,6 +18,7 @@ import {
 	ClipboardType,
 	DatabaseZap,
 	LayoutList,
+	Pyramid,
 	X,
 } from 'lucide-react';
 import { useCallback, useEffect, useState } from 'react';
@@ -34,6 +37,7 @@ import {
 } from '../../../DashboardVariablesSelection/util';
 import { variablePropsToPayloadVariables } from '../../../utils';
 import { TVariableMode } from '../types';
+import DynamicVariable from './DynamicVariable/DynamicVariable';
 import { LabelContainer, VariableItemRow } from './styles';
 
 const { Option } = Select;
@@ -85,11 +89,44 @@ function VariableItem({
 		variableData.showALLOption || false,
 	);
 	const [previewValues, setPreviewValues] = useState<string[]>([]);
+	const [variableDefaultValue, setVariableDefaultValue] = useState<string>(
+		(variableData.defaultValue as string) || '',
+	);
 
+	const [
+		dynamicVariablesSelectedValue,
+		setDynamicVariablesSelectedValue,
+	] = useState<{ name: string; value: string }>();
+
+	useEffect(() => {
+		if (
+			variableData.dynamicVariablesAttribute &&
+			variableData.dynamicVariablesSource
+		) {
+			setDynamicVariablesSelectedValue({
+				name: variableData.dynamicVariablesAttribute,
+				value: variableData.dynamicVariablesSource,
+			});
+		}
+	}, [
+		variableData.dynamicVariablesAttribute,
+		variableData.dynamicVariablesSource,
+	]);
 	// Error messages
 	const [errorName, setErrorName] = useState<boolean>(false);
 	const [errorPreview, setErrorPreview] = useState<string | null>(null);
 
+	const { data: fieldValues } = useGetFieldValues({
+		signal:
+			dynamicVariablesSelectedValue?.value === 'All Sources'
+				? undefined
+				: (dynamicVariablesSelectedValue?.value as 'traces' | 'logs' | 'metrics'),
+		name: dynamicVariablesSelectedValue?.name || '',
+		enabled:
+			!!dynamicVariablesSelectedValue?.name &&
+			!!dynamicVariablesSelectedValue?.value,
+	});
+
 	useEffect(() => {
 		if (queryType === 'CUSTOM') {
 			setPreviewValues(
@@ -110,6 +147,29 @@ function VariableItem({
 		variableSortType,
 	]);
 
+	useEffect(() => {
+		if (
+			queryType === 'DYNAMIC' &&
+			fieldValues &&
+			dynamicVariablesSelectedValue?.name &&
+			dynamicVariablesSelectedValue?.value
+		) {
+			setPreviewValues(
+				sortValues(
+					fieldValues.payload?.values?.stringValues || [],
+					variableSortType,
+				) as never,
+			);
+		}
+	}, [
+		fieldValues,
+		variableSortType,
+		queryType,
+		dynamicVariablesSelectedValue?.name,
+		dynamicVariablesSelectedValue?.value,
+		dynamicVariablesSelectedValue,
+	]);
+
 	const handleSave = (): void => {
 		// Check for cyclic dependencies
 		const newVariable = {
@@ -126,9 +186,16 @@ function VariableItem({
 				selectedValue: (variableData.selectedValue ||
 					variableTextboxValue) as never,
 			}),
+			...(queryType !== 'TEXTBOX' && {
+				defaultValue: variableDefaultValue as never,
+			}),
 			modificationUUID: generateUUID(),
 			id: variableData.id || generateUUID(),
 			order: variableData.order,
+			...(queryType === 'DYNAMIC' && {
+				dynamicVariablesAttribute: dynamicVariablesSelectedValue?.name,
+				dynamicVariablesSource: dynamicVariablesSelectedValue?.value,
+			}),
 		};
 
 		const allVariables = [...Object.values(existingVariables), newVariable];
@@ -258,18 +325,18 @@ function VariableItem({
 						<div className="variable-type-btn-group">
 							<Button
 								type="text"
-								icon={<DatabaseZap size={14} />}
+								icon={<Pyramid size={14} />}
 								className={cx(
 									// eslint-disable-next-line sonarjs/no-duplicate-string
 									'variable-type-btn',
-									queryType === 'QUERY' ? 'selected' : '',
+									queryType === 'DYNAMIC' ? 'selected' : '',
 								)}
 								onClick={(): void => {
-									setQueryType('QUERY');
+									setQueryType('DYNAMIC');
 									setPreviewValues([]);
 								}}
 							>
-								Query
+								Dynamic
 							</Button>
 							<Button
 								type="text"
@@ -299,8 +366,31 @@ function VariableItem({
 							>
 								Custom
 							</Button>
+							<Button
+								type="text"
+								icon={<DatabaseZap size={14} />}
+								className={cx(
+									// eslint-disable-next-line sonarjs/no-duplicate-string
+									'variable-type-btn',
+									queryType === 'QUERY' ? 'selected' : '',
+								)}
+								onClick={(): void => {
+									setQueryType('QUERY');
+									setPreviewValues([]);
+								}}
+							>
+								Query
+							</Button>
 						</div>
 					</VariableItemRow>
+					{queryType === 'DYNAMIC' && (
+						<div className="variable-dynamic-section">
+							<DynamicVariable
+								setDynamicVariablesSelectedValue={setDynamicVariablesSelectedValue}
+								dynamicVariablesSelectedValue={dynamicVariablesSelectedValue}
+							/>
+						</div>
+					)}
 					{queryType === 'QUERY' && (
 						<div className="query-container">
 							<LabelContainer>
@@ -388,7 +478,9 @@ function VariableItem({
 							/>
 						</VariableItemRow>
 					)}
-					{(queryType === 'QUERY' || queryType === 'CUSTOM') && (
+					{(queryType === 'QUERY' ||
+						queryType === 'CUSTOM' ||
+						queryType === 'DYNAMIC') && (
 						<>
 							<VariableItemRow className="variables-preview-section">
 								<LabelContainer style={{ width: '100%' }}>
@@ -457,6 +549,20 @@ function VariableItem({
 									/>
 								</VariableItemRow>
 							)}
+							<VariableItemRow className="default-value-section">
+								<LabelContainer>
+									<Typography className="typography-variables">Default Value</Typography>
+								</LabelContainer>
+								<CustomSelect
+									placeholder="Select a default value"
+									value={variableDefaultValue}
+									onChange={(value): void => setVariableDefaultValue(value)}
+									options={previewValues.map((value) => ({
+										label: value,
+										value,
+									}))}
+								/>
+							</VariableItemRow>
 						</>
 					)}
 				</div>

frontend/src/hooks/dynamicVariables/useGetFieldKeys.ts

@@ -0,0 +1,35 @@
+import { getFieldKeys } from 'api/dynamicVariables/getFieldKeys';
+import { useQuery, UseQueryResult } from 'react-query';
+import { ErrorResponse, SuccessResponse } from 'types/api';
+import { FieldKeyResponse } from 'types/api/dynamicVariables/getFieldKeys';
+
+interface UseGetFieldKeysProps {
+	/** Type of signal (traces, logs, metrics) */
+	signal?: 'traces' | 'logs' | 'metrics';
+	/** Optional search text */
+	name?: string;
+	/** Whether the query should be enabled */
+	enabled?: boolean;
+}
+
+/**
+ * Hook to fetch field keys for a given signal type
+ *
+ * If 'complete' in the response is true:
+ * - All subsequent searches should be local (client has complete list)
+ *
+ * If 'complete' is false:
+ * - All subsequent searches should use the API (passing the name param)
+ */
+export const useGetFieldKeys = ({
+	signal,
+	name,
+	enabled = true,
+}: UseGetFieldKeysProps): UseQueryResult<
+	SuccessResponse<FieldKeyResponse> | ErrorResponse
+> =>
+	useQuery<SuccessResponse<FieldKeyResponse> | ErrorResponse>({
+		queryKey: ['fieldKeys', signal, name],
+		queryFn: () => getFieldKeys(signal, name),
+		enabled,
+	});

frontend/src/hooks/dynamicVariables/useGetFieldValues.ts

@@ -0,0 +1,38 @@
+import { getFieldValues } from 'api/dynamicVariables/getFieldValues';
+import { useQuery, UseQueryResult } from 'react-query';
+import { ErrorResponse, SuccessResponse } from 'types/api';
+import { FieldValueResponse } from 'types/api/dynamicVariables/getFieldValues';
+
+interface UseGetFieldValuesProps {
+	/** Type of signal (traces, logs, metrics) */
+	signal?: 'traces' | 'logs' | 'metrics';
+	/** Name of the attribute for which values are being fetched */
+	name: string;
+	/** Optional search text */
+	value?: string;
+	/** Whether the query should be enabled */
+	enabled?: boolean;
+}
+
+/**
+ * Hook to fetch field values for a given signal type and field name
+ *
+ * If 'complete' in the response is true:
+ * - All subsequent searches should be local (client has complete list)
+ *
+ * If 'complete' is false:
+ * - All subsequent searches should use the API (passing the value param)
+ */
+export const useGetFieldValues = ({
+	signal,
+	name,
+	value,
+	enabled = true,
+}: UseGetFieldValuesProps): UseQueryResult<
+	SuccessResponse<FieldValueResponse> | ErrorResponse
+> =>
+	useQuery<SuccessResponse<FieldValueResponse> | ErrorResponse>({
+		queryKey: ['fieldValues', signal, name, value],
+		queryFn: () => getFieldValues(signal, name, value),
+		enabled,
+	});

frontend/src/types/api/dashboard/getAll.ts

@@ -9,7 +9,12 @@ import { Query } from 'types/api/queryBuilder/queryBuilderData';
 import { IField } from '../logs/fields';
 import { TelemetryFieldKey } from '../v5/queryRange';
 
-export const VariableQueryTypeArr = ['QUERY', 'TEXTBOX', 'CUSTOM'] as const;
+export const VariableQueryTypeArr = [
+	'QUERY',
+	'TEXTBOX',
+	'CUSTOM',
+	'DYNAMIC',
+] as const;
 export type TVariableQueryType = typeof VariableQueryTypeArr[number];
 
 export const VariableSortTypeArr = ['DISABLED', 'ASC', 'DESC'] as const;
@@ -46,6 +51,9 @@ export interface IDashboardVariable {
 	modificationUUID?: string;
 	allSelected?: boolean;
 	change?: boolean;
+	defaultValue?: string;
+	dynamicVariablesAttribute?: string;
+	dynamicVariablesSource?: string;
 }
 export interface Dashboard {
 	id: string;

frontend/src/types/api/dynamicVariables/getFieldKeys.ts

@@ -0,0 +1,23 @@
+/**
+ * Response from the field keys API
+ */
+export interface FieldKeyResponse {
+	/** List of field keys returned */
+	keys?: Record<string, FieldKey[]>;
+	/** Indicates if the returned list is complete */
+	complete?: boolean;
+}
+
+/**
+ * Field key data structure
+ */
+export interface FieldKey {
+	/** Key name */
+	name?: string;
+	/** Data type of the field */
+	fieldDataType?: string;
+	/** Signal type */
+	signal?: string;
+	/** Field context */
+	fieldContext?: string;
+}

frontend/src/types/api/dynamicVariables/getFieldValues.ts

@@ -0,0 +1,9 @@
+/**
+ * Response from the field values API
+ */
+export interface FieldValueResponse {
+	/** List of field values returned */
+	values: { stringValues: string[] };
+	/** Indicates if the returned list is complete */
+	complete: boolean;
+}