initial pose estimation drawing in ImageCanvas

Author: seveibar

src/ImageCanvas/dancing-man.story.jpg

@@ -10,7 +10,13 @@ import React, {
 import type { Node } from "react"
 import { Matrix } from "transformation-matrix-js"
 import Crosshairs from "../Crosshairs"
-import type { Region, Point, Polygon, Box } from "./region-tools.js"
+import type {
+  Region,
+  Point,
+  Polygon,
+  Box,
+  KeypointsDefinition,
+} from "./region-tools.js"
 import { makeStyles } from "@material-ui/core/styles"
 import styles from "./styles"
 import PreventScrollToParents from "../PreventScrollToParents"
@@ -36,6 +42,7 @@ type Props = {
   imageSrc?: string,
   videoSrc?: string,
   videoTime?: number,
+  keypointDefinitions?: KeypointDefinitions,
   onMouseMove?: ({ x: number, y: number }) => any,
   onMouseDown?: ({ x: number, y: number }) => any,
   onMouseUp?: ({ x: number, y: number }) => any,
@@ -128,6 +135,7 @@ export const ImageCanvas = ({
   onRegionClassAdded,
   zoomOnAllowedArea = true,
   modifyingAllowedArea = false,
+  keypointDefinitions,
 }: Props) => {
   const classes = useStyles()
 
@@ -440,6 +448,7 @@ export const ImageCanvas = ({
           />
           <RegionShapes
             mat={mat}
+            keypointDefinitions={keypointDefinitions}
             imagePosition={imagePosition}
             regions={regions}
             fullSegmentationMode={fullImageSegmentationMode}

src/ImageCanvas/index.js

@@ -1,5 +1,3 @@
-// @flow
-
 import React from "react"
 
 import { storiesOf } from "@storybook/react"
@@ -8,6 +6,7 @@ import { action } from "@storybook/addon-actions"
 import ImageCanvas from "./"
 import exampleMask from "./mouse_mask.story.png"
 import exampleImage from "./seves_desk.story.jpg"
+import dancingManImage from "./dancing-man.story.jpg"
 
 export const testRegions = [
   {
@@ -158,3 +157,56 @@ storiesOf("ImageCanvas", module)
       {...events}
     />
   ))
+  .add("Poses", () => (
+    <ImageCanvas
+      keypointDefinitions={{
+        human: {
+          connections: [
+            ["head", "torso"],
+            ["torso", "leftElbow"],
+            ["torso", "rightElbow"],
+          ],
+          landmarks: {
+            head: {
+              label: "Head",
+              color: "#f00",
+              defaultPosition: [0, -0.05],
+            },
+            torso: {
+              label: "Torso",
+              color: "#0f0",
+              defaultPosition: [0, 0],
+            },
+            leftElbow: {
+              label: "Left Elbow",
+              color: "#00f",
+              defaultPosition: [-0.05, 0],
+            },
+            rightElbow: {
+              label: "Right Elbow",
+              color: "#00f",
+              defaultPosition: [0.05, 0],
+            },
+          },
+        },
+      }}
+      regions={[
+        {
+          type: "keypoints",
+          id: "keypoints1",
+          keypointDefinitionId: "human",
+          highlighted: true,
+          points: {
+            head: [0.5, 0.4],
+            torso: [0.5, 0.5],
+            leftElbow: [0.4, 0.5],
+            rightElbow: [0.6, 0.5],
+          },
+          visible: true,
+        },
+      ]}
+      imageSrc={dancingManImage}
+      showTags
+      {...events}
+    />
+  ))

src/ImageCanvas/index.story.js

@@ -41,6 +41,7 @@ export type Box = {|
   w: number,
   h: number,
 |}
+
 export type Polygon = {|
   ...$Exact<BaseRegion>,
   type: "polygon",
@@ -53,7 +54,38 @@ export type ExpandingLine = {|
   points: Array<{ x: number, y: number, angle: number, width: number }>,
 |}
 
-export type Region = Point | PixelRegion | Box | Polygon | ExpandingLine
+export type Keypoint = {|
+  label: string,
+  defaultPosition: [number, number],
+|}
+
+export type KeypointId = string
+
+export type KeypointsDefinition = {|
+  [id: string]: {
+    connections: Array<[KeypointId, KeypointId]>,
+    landmarks: {
+      [KeypointId]: Keypoint,
+    },
+  },
+|}
+
+export type Keypoints = {|
+  ...$Exact<BaseRegion>,
+  type: "keypoints",
+  keypointsDefinitionId: string,
+  points: {
+    [string]: [number, number],
+  },
+|}
+
+export type Region =
+  | Point
+  | PixelRegion
+  | Box
+  | Polygon
+  | ExpandingLine
+  | Keypoints
 
 export const getEnclosingBox = (region: Region) => {
   switch (region.type) {

src/ImageCanvas/region-tools.js

@@ -46,6 +46,55 @@ const RegionComponents = {
       />
     )
   }),
+  keypoints: ({ region, iw, ih, keypointDefinitions }) => {
+    const { points, keypointDefinitionId } = region
+    if (!keypointDefinitions[keypointDefinitionId]) {
+      throw new Error(
+        `No definition for keypoint configuration "${keypointDefinitionId}"`
+      )
+    }
+    const { landmarks, connections } = keypointDefinitions[keypointDefinitionId]
+    return (
+      <g>
+        {Object.entries(points).map(([keypointId, [x, y]], i) => (
+          <g key={i} transform={`translate(${x * iw} ${y * ih})`}>
+            <path
+              d={"M0 8L8 0L0 -8L-8 0Z"}
+              strokeWidth={2}
+              stroke={landmarks[keypointId].color}
+              fill="transparent"
+            />
+          </g>
+        ))}
+        {connections.map(([kp1Id, kp2Id]) => {
+          const kp1 = points[kp1Id]
+          const kp2 = points[kp2Id]
+          const midPoint = [(kp1[0] + kp2[0]) / 2, (kp1[1] + kp2[1]) / 2]
+
+          return (
+            <g key={`${kp1}.${kp2}`}>
+              <line
+                x1={kp1[0] * iw}
+                y1={kp1[1] * ih}
+                x2={midPoint[0] * iw}
+                y2={midPoint[1] * ih}
+                strokeWidth={2}
+                stroke={landmarks[kp1Id].color}
+              />
+              <line
+                x1={kp2[0] * iw}
+                y1={kp2[1] * ih}
+                x2={midPoint[0] * iw}
+                y2={midPoint[1] * ih}
+                strokeWidth={2}
+                stroke={landmarks[kp2Id].color}
+              />
+            </g>
+          )
+        })}
+      </g>
+    )
+  },
   "expanding-line": memo(({ region, iw, ih }) => {
     let { expandingWidth = 0.005, points } = region
     expandingWidth = points.slice(-1)[0].width || expandingWidth
@@ -114,7 +163,7 @@ const RegionComponents = {
 }
 
 export const WrappedRegionList = memo(
-  ({ regions, iw, ih, fullSegmentationMode }) => {
+  ({ regions, keypointDefinitions, iw, ih, fullSegmentationMode }) => {
     return regions
       .filter((r) => r.visible !== false)
       .map((r) => {
@@ -125,7 +174,8 @@ export const WrappedRegionList = memo(
             region={r}
             iw={iw}
             ih={ih}
-            fullgeSegmentationMode={fullSegmentationMode}
+            keypointDefinitions={keypointDefinitions}
+            fullSegmentationMode={fullSegmentationMode}
           />
         )
       })
@@ -137,6 +187,7 @@ export const RegionShapes = ({
   mat,
   imagePosition,
   regions = [],
+  keypointDefinitions,
   fullSegmentationMode,
 }) => {
   const iw = imagePosition.bottomRight.x - imagePosition.topLeft.x
@@ -161,6 +212,7 @@ export const RegionShapes = ({
         regions={regions}
         iw={iw}
         ih={ih}
+        keypointDefinitions={keypointDefinitions}
         fullSegmentationMode={fullSegmentationMode}
       />
     </svg>