ui: controlnet in linear ui

invokeai/app/invocations/controlnet_image_processors.py

@@ -94,13 +94,13 @@
 CONTROLNET_NAME_VALUES = Literal[tuple(CONTROLNET_DEFAULT_MODELS)]
 
 class ControlField(BaseModel):
-    image: ImageField = Field(default=None, description="processed image")
-    control_model: Optional[str] = Field(default=None, description="control model used")
-    control_weight: Optional[float] = Field(default=1, description="weight given to controlnet")
+    image: ImageField = Field(default=None, description="The control image")
+    control_model: Optional[str] = Field(default=None, description="The ControlNet model to use")
+    control_weight: Optional[float] = Field(default=1, description="The weight given to the ControlNet")
     begin_step_percent: float = Field(default=0, ge=0, le=1,
-                                                description="% of total steps at which controlnet is first applied")
+                                                description="When the ControlNet is first applied (% of total steps)")
     end_step_percent: float = Field(default=1, ge=0, le=1,
-                                    description="% of total steps at which controlnet is last applied")
+                                    description="When the ControlNet is last applied (% of total steps)")
 
     class Config:
         schema_extra = {
@@ -112,7 +112,7 @@ class ControlOutput(BaseInvocationOutput):
     """node output for ControlNet info"""
     # fmt: off
     type: Literal["control_output"] = "control_output"
-    control: ControlField = Field(default=None, description="The control info dict")
+    control: ControlField = Field(default=None, description="The output control image")
     # fmt: on
 
 
@@ -121,15 +121,15 @@ class ControlNetInvocation(BaseInvocation):
     # fmt: off
     type: Literal["controlnet"] = "controlnet"
     # Inputs
-    image: ImageField = Field(default=None, description="image to process")
+    image: ImageField = Field(default=None, description="The control image")
     control_model: CONTROLNET_NAME_VALUES = Field(default="lllyasviel/sd-controlnet-canny",
-                                                  description="control model used")
-    control_weight: float = Field(default=1.0, ge=0, le=1, description="weight given to controlnet")
+                                                  description="The ControlNet model to use")
+    control_weight: float = Field(default=1.0, ge=0, le=1, description="The weight given to the ControlNet")
     # TODO: add support in backend core for begin_step_percent, end_step_percent, guess_mode
     begin_step_percent: float = Field(default=0, ge=0, le=1,
-                                        description="% of total steps at which controlnet is first applied")
+                                        description="When the ControlNet is first applied (% of total steps)")
     end_step_percent: float = Field(default=1, ge=0, le=1,
-                                      description="% of total steps at which controlnet is last applied")
+                                      description="When the ControlNet is last applied (% of total steps)")
     # fmt: on
 
 
@@ -152,7 +152,7 @@ class ImageProcessorInvocation(BaseInvocation, PILInvocationConfig):
     # fmt: off
     type: Literal["image_processor"] = "image_processor"
     # Inputs
-    image: ImageField = Field(default=None, description="image to process")
+    image: ImageField = Field(default=None, description="The image to process")
     # fmt: on
 
 
@@ -204,8 +204,8 @@ class CannyImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfi
     # fmt: off
     type: Literal["canny_image_processor"] = "canny_image_processor"
     # Input
-    low_threshold: float = Field(default=100, ge=0, description="low threshold of Canny pixel gradient")
-    high_threshold: float = Field(default=200, ge=0, description="high threshold of Canny pixel gradient")
+    low_threshold: int = Field(default=100, ge=0, le=255, description="The low threshold of the Canny pixel gradient (0-255)")
+    high_threshold: int = Field(default=200, ge=0, le=255, description="The high threshold of the Canny pixel gradient (0-255)")
     # fmt: on
 
     def run_processor(self, image):
@@ -214,16 +214,16 @@ def run_processor(self, image):
         return processed_image
 
 
-class HedImageprocessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
+class HedImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig):
     """Applies HED edge detection to image"""
     # fmt: off
     type: Literal["hed_image_processor"] = "hed_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
     # safe not supported in controlnet_aux v0.0.3
     # safe: bool = Field(default=False, description="whether to use safe mode")
-    scribble: bool = Field(default=False, description="whether to use scribble mode")
+    scribble: bool = Field(default=False, description="Whether to use scribble mode")
     # fmt: on
 
     def run_processor(self, image):
@@ -243,9 +243,9 @@ class LineartImageProcessorInvocation(ImageProcessorInvocation, PILInvocationCon
     # fmt: off
     type: Literal["lineart_image_processor"] = "lineart_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
-    coarse: bool = Field(default=False, description="whether to use coarse mode")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
+    coarse: bool = Field(default=False, description="Whether to use coarse mode")
     # fmt: on
 
     def run_processor(self, image):
@@ -262,8 +262,8 @@ class LineartAnimeImageProcessorInvocation(ImageProcessorInvocation, PILInvocati
     # fmt: off
     type: Literal["lineart_anime_image_processor"] = "lineart_anime_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
     # fmt: on
 
     def run_processor(self, image):
@@ -280,9 +280,9 @@ class OpenposeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationCo
     # fmt: off
     type: Literal["openpose_image_processor"] = "openpose_image_processor"
     # Inputs
-    hand_and_face: bool = Field(default=False, description="whether to use hands and face mode")
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
+    hand_and_face: bool = Field(default=False, description="Whether to use hands and face mode")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
     # fmt: on
 
     def run_processor(self, image):
@@ -300,8 +300,8 @@ class MidasDepthImageProcessorInvocation(ImageProcessorInvocation, PILInvocation
     # fmt: off
     type: Literal["midas_depth_image_processor"] = "midas_depth_image_processor"
     # Inputs
-    a_mult: float = Field(default=2.0, ge=0, description="Midas parameter a = amult * PI")
-    bg_th: float = Field(default=0.1, ge=0, description="Midas parameter bg_th")
+    a_mult: float = Field(default=2.0, ge=0, description="Midas parameter `a_mult` (a = a_mult * PI)")
+    bg_th: float = Field(default=0.1, ge=0, description="Midas parameter `bg_th`")
     # depth_and_normal not supported in controlnet_aux v0.0.3
     # depth_and_normal: bool = Field(default=False, description="whether to use depth and normal mode")
     # fmt: on
@@ -322,8 +322,8 @@ class NormalbaeImageProcessorInvocation(ImageProcessorInvocation, PILInvocationC
     # fmt: off
     type: Literal["normalbae_image_processor"] = "normalbae_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
     # fmt: on
 
     def run_processor(self, image):
@@ -339,10 +339,10 @@ class MlsdImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig
     # fmt: off
     type: Literal["mlsd_image_processor"] = "mlsd_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
-    thr_v: float = Field(default=0.1, ge=0, description="MLSD parameter thr_v")
-    thr_d: float = Field(default=0.1, ge=0, description="MLSD parameter thr_d")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
+    thr_v: float = Field(default=0.1, ge=0, description="MLSD parameter `thr_v`")
+    thr_d: float = Field(default=0.1, ge=0, description="MLSD parameter `thr_d`")
     # fmt: on
 
     def run_processor(self, image):
@@ -360,10 +360,10 @@ class PidiImageProcessorInvocation(ImageProcessorInvocation, PILInvocationConfig
     # fmt: off
     type: Literal["pidi_image_processor"] = "pidi_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
-    safe: bool = Field(default=False, description="whether to use safe mode")
-    scribble: bool = Field(default=False, description="whether to use scribble mode")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
+    safe: bool = Field(default=False, description="Whether to use safe mode")
+    scribble: bool = Field(default=False, description="Whether to use scribble mode")
     # fmt: on
 
     def run_processor(self, image):
@@ -381,11 +381,11 @@ class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation, PILInvoca
     # fmt: off
     type: Literal["content_shuffle_image_processor"] = "content_shuffle_image_processor"
     # Inputs
-    detect_resolution: int = Field(default=512, ge=0, description="pixel resolution for edge detection")
-    image_resolution: int = Field(default=512, ge=0, description="pixel resolution for output image")
-    h: Union[int | None] = Field(default=512, ge=0, description="content shuffle h parameter")
-    w: Union[int | None] = Field(default=512, ge=0, description="content shuffle w parameter")
-    f: Union[int | None] = Field(default=256, ge=0, description="cont")
+    detect_resolution: int = Field(default=512, ge=0, description="The pixel resolution for detection")
+    image_resolution: int = Field(default=512, ge=0, description="The pixel resolution for the output image")
+    h: Union[int, None] = Field(default=512, ge=0, description="Content shuffle `h` parameter")
+    w: Union[int, None] = Field(default=512, ge=0, description="Content shuffle `w` parameter")
+    f: Union[int, None] = Field(default=256, ge=0, description="Content shuffle `f` parameter")
     # fmt: on
 
     def run_processor(self, image):
@@ -418,8 +418,8 @@ class MediapipeFaceProcessorInvocation(ImageProcessorInvocation, PILInvocationCo
     # fmt: off
     type: Literal["mediapipe_face_processor"] = "mediapipe_face_processor"
     # Inputs
-    max_faces: int = Field(default=1, ge=1, description="maximum number of faces to detect")
-    min_confidence: float = Field(default=0.5, ge=0, le=1, description="minimum confidence for face detection")
+    max_faces: int = Field(default=1, ge=1, description="Maximum number of faces to detect")
+    min_confidence: float = Field(default=0.5, ge=0, le=1, description="Minimum confidence for face detection")
     # fmt: on
 
     def run_processor(self, image):

invokeai/frontend/web/package.json

@@ -60,6 +60,8 @@
     "@chakra-ui/styled-system": "^2.9.0",
     "@chakra-ui/theme-tools": "^2.0.16",
     "@dagrejs/graphlib": "^2.1.12",
+    "@dnd-kit/core": "^6.0.8",
+    "@dnd-kit/modifiers": "^6.0.1",
     "@emotion/react": "^11.10.6",
     "@emotion/styled": "^11.10.6",
     "@floating-ui/react-dom": "^2.0.0",
@@ -87,7 +89,7 @@
     "react-dropzone": "^14.2.3",
     "react-hotkeys-hook": "4.4.0",
     "react-i18next": "^12.2.2",
-    "react-icons": "^4.7.1",
+    "react-icons": "^4.9.0",
     "react-konva": "^18.2.7",
     "react-redux": "^8.0.5",
     "react-resizable-panels": "^0.0.42",

invokeai/frontend/web/src/app/components/ImageDnd/ImageDndContext.tsx

@@ -0,0 +1,68 @@
+import {
+  DndContext,
+  DragEndEvent,
+  DragOverlay,
+  DragStartEvent,
+  KeyboardSensor,
+  MouseSensor,
+  TouchSensor,
+  pointerWithin,
+  useSensor,
+  useSensors,
+} from '@dnd-kit/core';
+import { PropsWithChildren, memo, useCallback, useState } from 'react';
+import OverlayDragImage from './OverlayDragImage';
+import { ImageDTO } from 'services/api';
+import { isImageDTO } from 'services/types/guards';
+import { snapCenterToCursor } from '@dnd-kit/modifiers';
+
+type ImageDndContextProps = PropsWithChildren;
+
+const ImageDndContext = (props: ImageDndContextProps) => {
+  const [draggedImage, setDraggedImage] = useState<ImageDTO | null>(null);
+
+  const handleDragStart = useCallback((event: DragStartEvent) => {
+    const dragData = event.active.data.current;
+    if (dragData && 'image' in dragData && isImageDTO(dragData.image)) {
+      setDraggedImage(dragData.image);
+    }
+  }, []);
+
+  const handleDragEnd = useCallback(
+    (event: DragEndEvent) => {
+      const handleDrop = event.over?.data.current?.handleDrop;
+      if (handleDrop && typeof handleDrop === 'function' && draggedImage) {
+        handleDrop(draggedImage);
+      }
+      setDraggedImage(null);
+    },
+    [draggedImage]
+  );
+
+  const mouseSensor = useSensor(MouseSensor, {
+    activationConstraint: { distance: 15 },
+  });
+
+  const touchSensor = useSensor(TouchSensor, {
+    activationConstraint: { distance: 15 },
+  });
+  const keyboardSensor = useSensor(KeyboardSensor);
+
+  const sensors = useSensors(mouseSensor, touchSensor, keyboardSensor);
+
+  return (
+    <DndContext
+      onDragStart={handleDragStart}
+      onDragEnd={handleDragEnd}
+      sensors={sensors}
+      collisionDetection={pointerWithin}
+    >
+      {props.children}
+      <DragOverlay dropAnimation={null} modifiers={[snapCenterToCursor]}>
+        {draggedImage && <OverlayDragImage image={draggedImage} />}
+      </DragOverlay>
+    </DndContext>
+  );
+};
+
+export default memo(ImageDndContext);

invokeai/frontend/web/src/app/components/ImageDnd/OverlayDragImage.tsx

@@ -0,0 +1,36 @@
+import { Box, Image } from '@chakra-ui/react';
+import { memo } from 'react';
+import { ImageDTO } from 'services/api';
+
+type OverlayDragImageProps = {
+  image: ImageDTO;
+};
+
+const OverlayDragImage = (props: OverlayDragImageProps) => {
+  return (
+    <Box
+      style={{
+        width: '100%',
+        height: '100%',
+        display: 'flex',
+        alignItems: 'center',
+        justifyContent: 'center',
+        userSelect: 'none',
+        cursor: 'grabbing',
+        opacity: 0.5,
+      }}
+    >
+      <Image
+        sx={{
+          maxW: 36,
+          maxH: 36,
+          borderRadius: 'base',
+          shadow: 'dark-lg',
+        }}
+        src={props.image.thumbnail_url}
+      />
+    </Box>
+  );
+};
+
+export default memo(OverlayDragImage);

invokeai/frontend/web/src/app/components/InvokeAIUI.tsx

@@ -16,6 +16,7 @@ import { PartialAppConfig } from 'app/types/invokeai';
 import '../../i18n';
 import { socketMiddleware } from 'services/events/middleware';
 import { Middleware } from '@reduxjs/toolkit';
+import ImageDndContext from './ImageDnd/ImageDndContext';
 
 const App = lazy(() => import('./App'));
 const ThemeLocaleProvider = lazy(() => import('./ThemeLocaleProvider'));
@@ -69,11 +70,13 @@ const InvokeAIUI = ({
       <Provider store={store}>
         <React.Suspense fallback={<Loading />}>
           <ThemeLocaleProvider>
-            <App
-              config={config}
-              headerComponent={headerComponent}
-              setIsReady={setIsReady}
-            />
+            <ImageDndContext>
+              <App
+                config={config}
+                headerComponent={headerComponent}
+                setIsReady={setIsReady}
+              />
+            </ImageDndContext>
           </ThemeLocaleProvider>
         </React.Suspense>
       </Provider>

invokeai/frontend/web/src/app/store/enhancers/reduxRemember/serialize.ts

@@ -1,4 +1,5 @@
 import { canvasPersistDenylist } from 'features/canvas/store/canvasPersistDenylist';
+import { controlNetDenylist } from 'features/controlNet/store/controlNetDenylist';
 import { galleryPersistDenylist } from 'features/gallery/store/galleryPersistDenylist';
 import { lightboxPersistDenylist } from 'features/lightbox/store/lightboxPersistDenylist';
 import { nodesPersistDenylist } from 'features/nodes/store/nodesPersistDenylist';
@@ -23,6 +24,7 @@ const serializationDenylist: {
   system: systemPersistDenylist,
   // config: configPersistDenyList,
   ui: uiPersistDenylist,
+  controlNet: controlNetDenylist,
   // hotkeys: hotkeysPersistDenylist,
 };
 

invokeai/frontend/web/src/app/store/enhancers/reduxRemember/unserialize.ts

@@ -1,4 +1,5 @@
 import { initialCanvasState } from 'features/canvas/store/canvasSlice';
+import { initialControlNetState } from 'features/controlNet/store/controlNetSlice';
 import { initialGalleryState } from 'features/gallery/store/gallerySlice';
 import { initialImagesState } from 'features/gallery/store/imagesSlice';
 import { initialLightboxState } from 'features/lightbox/store/lightboxSlice';
@@ -28,6 +29,7 @@ const initialStates: {
   ui: initialUIState,
   hotkeys: initialHotkeysState,
   images: initialImagesState,
+  controlNet: initialControlNetState,
 };
 
 export const unserialize: UnserializeFunction = (data, key) => {