[ { "id": "case01", "tree": "robodojo_lerobot_v3/robodojo_sim_arx_x5", "episode": 0, "fps": 25, "n_frames": 578, "l3": "Arrange the numbers from left to right to form the largest possible number, and place them on the pad.", "camera": "observation.images.cam_high", "source_video": "videos/observation.images.cam_high/chunk-000/file-000.mp4", "source_video_start_s": 0.0, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 28, "verb": "reach", "text": "Move the left arm towards the yellow number 0 on the left side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 96, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 28, "end": 47, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 47, "end": 90, "verb": "place", "text": "Place the held number 0 onto the leftmost slot of the pad.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 90, "end": 124, "verb": "retreat", "text": "Retract the left arm away from the pad.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 96, "end": 123, "verb": "reach", "text": "Move the right arm towards the yellow number 5 block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 123, "end": 145, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 124, "end": 190, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 145, "end": 185, "verb": "place", "text": "Place the held number 5 block onto the table surface.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 185, "end": 219, "verb": "retreat", "text": "Retreat the right arm away from the placed block.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 190, "end": 231, "verb": "grasp", "text": "Grasp the yellow number 5 located near the center of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 219, "end": 271, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 231, "end": 265, "verb": "place", "text": "Place the held number 5 onto the second slot of the pad.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 7, "start": 265, "end": 300, "verb": "retreat", "text": "Move the left arm back to its resting position.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 6, "start": 271, "end": 295, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 7, "start": 295, "end": 321, "verb": "grasp", "text": "Grasp the yellow number 2 block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 8, "start": 300, "end": 370, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 8, "start": 321, "end": 363, "verb": "place", "text": "Place the number 2 block to the left of the number 5 block.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 9, "start": 363, "end": 399, "verb": "retreat", "text": "Move the right arm back and to the right.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 9, "start": 370, "end": 395, "verb": "reach", "text": "Reach towards the yellow number 2 on the right side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 10, "start": 395, "end": 420, "verb": "grasp", "text": "Grasp the yellow number 2.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 10, "start": 399, "end": 471, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 11, "start": 420, "end": 466, "verb": "place", "text": "Place the number block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 12, "start": 466, "end": 500, "verb": "retreat", "text": "Move the left arm away from the number blocks.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 11, "start": 471, "end": 511, "verb": "grasp", "text": "Grasp the yellow number 0 block located further to the left.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 13, "start": 500, "end": 578, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 12, "start": 511, "end": 547, "verb": "place", "text": "Place the number 0 block to the left of the number 2 block.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 13, "start": 547, "end": 578, "verb": "retreat", "text": "Move the right arm away from the number blocks on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 124, "t0": 0.0, "t1": 4.96, "l1_refs": { "left": [ 0, 1, 2, 3 ], "right": [ 0, 1 ] }, "text": "Place the number 0 block on the leftmost slot of the pad.", "verbs": [ "reach", "place", "retreat" ], "objects": [ "number block" ], "arms": [ "left" ], "flags": [] }, { "idx": 1, "start": 124, "end": 219, "t0": 4.96, "t1": 8.76, "l1_refs": { "left": [ 4, 5 ], "right": [ 2, 3, 4 ] }, "text": "Move the number 5 block to a temporary position on the table.", "verbs": [ "place", "retreat", "grasp" ], "objects": [ "number block" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 2, "start": 219, "end": 300, "t0": 8.76, "t1": 12.0, "l1_refs": { "left": [ 6, 7 ], "right": [ 5, 6 ] }, "text": "Place the number 5 block on the second slot of the pad.", "verbs": [ "place", "retreat" ], "objects": [ "number block" ], "arms": [ "left" ], "flags": [ "splits_l1" ] }, { "idx": 3, "start": 300, "end": 399, "t0": 12.0, "t1": 15.96, "l1_refs": { "left": [ 8, 9 ], "right": [ 7, 8, 9 ] }, "text": "Move the number 2 block to a temporary position on the table.", "verbs": [ "grasp", "place", "retreat", "reach" ], "objects": [ "number block" ], "arms": [ "right" ], "flags": [ "splits_l1" ] }, { "idx": 4, "start": 399, "end": 500, "t0": 15.96, "t1": 20.0, "l1_refs": { "left": [ 10, 11, 12 ], "right": [ 10, 11 ] }, "text": "Reposition the number 2 block on the table.", "verbs": [ "grasp", "place", "retreat" ], "objects": [ "number block" ], "arms": [ "left", "right" ], "flags": [ "splits_l1" ] }, { "idx": 5, "start": 500, "end": 578, "t0": 20.0, "t1": 23.12, "l1_refs": { "left": [ 13 ], "right": [ 12, 13 ] }, "text": "Arrange the remaining number blocks on the table to form the largest possible number.", "verbs": [ "place", "retreat" ], "objects": [ "number block" ], "arms": [ "right" ], "flags": [ "splits_l1" ] } ], "duration_s": 23.12 }, { "id": "case02", "tree": "robodojo_lerobot_v3/robodojo_sim_arx_x5", "episode": 3000, "fps": 25, "n_frames": 240, "l3": "Stack the three bowls together.", "camera": "observation.images.cam_high", "source_video": "videos/observation.images.cam_high/chunk-000/file-123.mp4", "source_video_start_s": 498.0799999999998, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 111, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 61, "verb": "grasp", "text": "Move the right arm towards the rightmost bowl and grasp it.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 61, "end": 110, "verb": "place", "text": "Place the held bowl on top of the middle bowl to stack them.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 110, "end": 152, "verb": "retreat", "text": "Retreat the right arm away from the stacked bowls.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 111, "end": 159, "verb": "grasp", "text": "Move the left arm to the single bowl on the left and grasp it by the rim.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 152, "end": 240, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 159, "end": 226, "verb": "place", "text": "Lift the held bowl, move it over the stack of bowls in the center, and place it on top.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": "持物动词，夹爪张开", "upstream_compat": null, "signals": [ "s_g2_heldverb_open" ], "mask_default": false, "mask_strict": true }, "compat": false, "grip_flag": "持物动词，夹爪张开" }, { "arm": "left", "idx": 3, "start": 226, "end": 240, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 111, "t0": 0.0, "t1": 4.44, "l1_refs": { "left": [ 0 ], "right": [ 0, 1 ] }, "text": "Grasp the rightmost bowl with the right arm and place it on top of the middle bowl.", "verbs": [ "grasp", "place" ], "objects": [ "bowl" ], "arms": [ "right" ], "flags": [] }, { "idx": 1, "start": 111, "end": 240, "t0": 4.44, "t1": 9.6, "l1_refs": { "left": [ 1, 2, 3 ], "right": [ 2, 3 ] }, "text": "Grasp the left bowl with the left arm and stack it on top of the existing pile.", "verbs": [ "retreat", "grasp", "place" ], "objects": [ "bowl" ], "arms": [ "left" ], "flags": [ "splits_l1" ] } ], "duration_s": 9.6 }, { "id": "case03", "tree": "robodojo_lerobot_v3/robodojo_sim_arx_x5", "episode": 600, "fps": 25, "n_frames": 1510, "l3": "Insert and tighten each screw into the nut of the same color.", "camera": "observation.images.cam_high", "source_video": "videos/observation.images.cam_high/chunk-000/file-028.mp4", "source_video_start_s": 157.48000000000002, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 91, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 39, "verb": "grasp", "text": "Grasp the blue screw on the right side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 39, "end": 85, "verb": "place", "text": "Place the blue screw onto the white nut in the center of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 85, "end": 121, "verb": "retreat", "text": "Retreat the arm away from the nut and screw assembly.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 91, "end": 133, "verb": "grasp", "text": "Grasp the blue screw on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 121, "end": 593, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 133, "end": 172, "verb": "place", "text": "Place the blue screw into the white nut.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 172, "end": 443, "verb": "rotate", "text": "Rotate the blue screw to tighten it into the white nut.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 443, "end": 477, "verb": "release", "text": "Release the blue screw after tightening it.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 477, "end": 495, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 495, "end": 544, "verb": "grasp", "text": "Grasp the red screw on the left side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 7, "start": 544, "end": 586, "verb": "place", "text": "Place the red screw into the white nut.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 8, "start": 586, "end": 622, "verb": "retreat", "text": "Move the arm away from the assembly area.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 593, "end": 634, "verb": "grasp", "text": "Grasp the red screw located near the left side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 9, "start": 622, "end": 1007, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 634, "end": 674, "verb": "release", "text": "Release the red screw onto the table surface.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 6, "start": 674, "end": 946, "verb": "grasp", "text": "Grasp the red screw again.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 7, "start": 946, "end": 979, "verb": "rotate", "text": "Rotate the held red screw to tighten it into the nut.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 8, "start": 979, "end": 997, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 9, "start": 997, "end": 1033, "verb": "retreat", "text": "Move the arm backward and to the right away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 10, "start": 1007, "end": 1048, "verb": "grasp", "text": "Grasp the red screw located at the back left of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 10, "start": 1033, "end": 1097, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 11, "start": 1048, "end": 1091, "verb": "place", "text": "Place the red screw on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 12, "start": 1091, "end": 1127, "verb": "retreat", "text": "Retreat the left arm away from the objects on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 11, "start": 1097, "end": 1140, "verb": "grasp", "text": "Move the arm to the red screw on the table and grasp it.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 13, "start": 1127, "end": 1510, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 12, "start": 1140, "end": 1178, "verb": "transport", "text": "Carry the held red screw towards the white nut.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 13, "start": 1178, "end": 1214, "verb": "insert", "text": "Insert the red screw into the white nut.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 14, "start": 1214, "end": 1428, "verb": "rotate", "text": "Rotate the wrist to tighten the screw into the nut.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 15, "start": 1428, "end": 1461, "verb": "release", "text": "Release the grip on the screw.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 16, "start": 1461, "end": 1479, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 17, "start": 1479, "end": 1510, "verb": "retreat", "text": "Retreat the arm away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 121, "t0": 0.0, "t1": 4.84, "l1_refs": { "left": [ 0, 1 ], "right": [ 0, 1, 2 ] }, "text": "Place the blue screw onto the white nut.", "verbs": [ "grasp", "place", "retreat" ], "objects": [ "screw", "blue screw" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 1, "start": 121, "end": 495, "t0": 4.84, "t1": 19.8, "l1_refs": { "left": [ 2, 3, 4, 5 ], "right": [ 3 ] }, "text": "Insert and tighten the blue screw into the white nut.", "verbs": [ "place", "rotate", "release" ], "objects": [ "blue screw" ], "arms": [ "left" ], "flags": [ "splits_l1" ] }, { "idx": 2, "start": 495, "end": 622, "t0": 19.8, "t1": 24.88, "l1_refs": { "left": [ 6, 7, 8 ], "right": [ 4 ] }, "text": "Place the red screw into the white nut.", "verbs": [ "grasp", "place", "retreat" ], "objects": [ "red screw", "screw" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 3, "start": 622, "end": 997, "t0": 24.88, "t1": 39.88, "l1_refs": { "left": [ 9 ], "right": [ 5, 6, 7, 8 ] }, "text": "Tighten the red screw into the white nut.", "verbs": [ "release", "grasp", "rotate" ], "objects": [ "screw" ], "arms": [ "right" ], "flags": [ "splits_l1" ] }, { "idx": 4, "start": 997, "end": 1127, "t0": 39.88, "t1": 45.08, "l1_refs": { "left": [ 10, 11, 12 ], "right": [ 9, 10, 11 ] }, "text": "Move the red screw to the table surface.", "verbs": [ "retreat", "grasp", "place" ], "objects": [ "red screw", "screw" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 5, "start": 1127, "end": 1510, "t0": 45.08, "t1": 60.4, "l1_refs": { "left": [ 13 ], "right": [ 12, 13, 14, 15, 16, 17 ] }, "text": "Insert and tighten the red screw into the white nut.", "verbs": [ "transport", "insert", "rotate", "release", "retreat" ], "objects": [ "screw" ], "arms": [ "right" ], "flags": [ "splits_l1" ] } ], "duration_s": 60.4 }, { "id": "case04", "tree": "robodojo_lerobot_v3/robodojo_sim_arx_x5", "episode": 1300, "fps": 25, "n_frames": 319, "l3": "Insert the three tubes into the rack one by one.", "camera": "observation.images.cam_high", "source_video": "videos/observation.images.cam_high/chunk-000/file-065.mp4", "source_video_start_s": 143.64000000000004, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 39, "verb": "grasp", "text": "Grasp the test tube on the left side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 98, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 39, "end": 90, "verb": "place", "text": "Place the test tube into the blue tube rack.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 90, "end": 124, "verb": "retreat", "text": "Retreat the arm away from the tube rack.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 98, "end": 138, "verb": "grasp", "text": "Grasp the test tube lying on the right side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 124, "end": 198, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 138, "end": 190, "verb": "insert", "text": "Insert the held test tube into an empty slot in the blue tube rack.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 190, "end": 224, "verb": "retreat", "text": "Move the arm backward and to the right away from the tube rack.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 198, "end": 237, "verb": "grasp", "text": "Grasp the second test tube located near the center of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 224, "end": 319, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 237, "end": 289, "verb": "place", "text": "Place the test tube into the blue tube rack.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 289, "end": 319, "verb": "retreat", "text": "Retreat the arm away from the tube rack.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 124, "t0": 0.0, "t1": 4.96, "l1_refs": { "left": [ 0, 1, 2 ], "right": [ 0, 1 ] }, "text": "Insert the first test tube into the blue tube rack.", "verbs": [ "grasp", "place", "retreat" ], "objects": [ "test tube" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 1, "start": 124, "end": 224, "t0": 4.96, "t1": 8.96, "l1_refs": { "left": [ 3, 4 ], "right": [ 2, 3 ] }, "text": "Insert the second test tube into the blue tube rack.", "verbs": [ "insert", "retreat", "grasp" ], "objects": [ "test tube" ], "arms": [ "left", "right" ], "flags": [ "splits_l1" ] }, { "idx": 2, "start": 224, "end": 319, "t0": 8.96, "t1": 12.76, "l1_refs": { "left": [ 5, 6 ], "right": [ 4 ] }, "text": "Insert the third test tube into the blue tube rack.", "verbs": [ "place", "retreat" ], "objects": [ "test tube" ], "arms": [ "left" ], "flags": [ "splits_l1" ] } ], "duration_s": 12.76 }, { "id": "case05", "tree": "robodojo_lerobot_v3/robodojo_sim_arx_x5", "episode": 900, "fps": 25, "n_frames": 300, "l3": "Fold the clothes neatly.", "camera": "observation.images.cam_high", "source_video": "videos/observation.images.cam_high/chunk-000/file-047.mp4", "source_video_start_s": 72.75999999999999, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 46, "verb": "grasp", "text": "Grasp the left sleeve of the long-sleeve top.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 95, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 46, "end": 89, "verb": "fold", "text": "Fold the left sleeve over the body of the shirt.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 89, "end": 124, "verb": "retreat", "text": "Move the left arm away from the folded shirt.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 95, "end": 149, "verb": "grasp", "text": "Grasp the right sleeve of the long-sleeve top.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 124, "end": 195, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 149, "end": 191, "verb": "fold", "text": "Fold the right sleeve over the body of the shirt.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 191, "end": 234, "verb": "grasp", "text": "Grasp the bottom edge of the long-sleeve top.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 195, "end": 234, "verb": "grasp", "text": "Grasp the bottom edge of the long-sleeve top.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 234, "end": 280, "verb": "fold", "text": "Fold the bottom of the shirt upwards towards the collar.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 234, "end": 280, "verb": "fold", "text": "Fold the bottom of the shirt upwards towards the collar.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 280, "end": 300, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 280, "end": 300, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 124, "t0": 0.0, "t1": 4.96, "l1_refs": { "left": [ 0, 1, 2 ], "right": [ 0, 1 ] }, "text": "Fold the left sleeve of the long-sleeve top over the body.", "verbs": [ "grasp", "fold", "retreat" ], "objects": [ "long-sleeve top" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 1, "start": 124, "end": 191, "t0": 4.96, "t1": 7.64, "l1_refs": { "left": [ 3 ], "right": [ 2 ] }, "text": "Fold the right sleeve of the long-sleeve top over the body.", "verbs": [ "fold" ], "objects": [ "long-sleeve top" ], "arms": [ "right" ], "flags": [ "splits_l1" ] }, { "idx": 2, "start": 191, "end": 300, "t0": 7.64, "t1": 12.0, "l1_refs": { "left": [ 4, 5, 6 ], "right": [ 3, 4, 5 ] }, "text": "Fold the bottom edge of the long-sleeve top upwards towards the collar.", "verbs": [ "grasp", "fold" ], "objects": [ "long-sleeve top" ], "arms": [ "left", "right" ], "flags": [] } ], "duration_s": 12.0 }, { "id": "case06", "tree": "robodojo_lerobot_v3/robodojo_sim_arx_x5", "episode": 1700, "fps": 25, "n_frames": 838, "l3": "Place the alarm clock on the drawer, put the figurine on the stand, place the mouse on the mouse pad, and push the keyboard into the frame.", "camera": "observation.images.cam_high", "source_video": "videos/observation.images.cam_high/chunk-000/file-078.mp4", "source_video_start_s": 236.44, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 69, "verb": "grasp", "text": "Grasp the alarm clock on the desk.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 367, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 69, "end": 123, "verb": "place", "text": "Place the alarm clock on top of the drawer unit.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 123, "end": 198, "verb": "grasp", "text": "Grasp the green figurine on the desk.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 198, "end": 354, "verb": "place", "text": "Place the figurine onto the black stand.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 354, "end": 402, "verb": "retreat", "text": "Retreat the arm away from the stand.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 367, "end": 442, "verb": "grasp", "text": "Grasp the white mouse on the desk.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 402, "end": 552, "verb": "grasp", "text": "Grasp the left edge of the white keyboard.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 442, "end": 492, "verb": "place", "text": "Place the mouse onto the black mouse pad.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 492, "end": 549, "verb": "grasp", "text": "Grasp the right edge of the white keyboard.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 549, "end": 604, "verb": "push", "text": "Push the keyboard forward towards the frame.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 552, "end": 582, "verb": "push", "text": "Push the keyboard towards the right to align it within the frame.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 7, "start": 582, "end": 603, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 8, "start": 603, "end": 838, "verb": "release", "text": "Release the keyboard and retract the arm.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 604, "end": 639, "verb": "hold", "text": "Hold the keyboard steady while the left arm adjusts it.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 6, "start": 639, "end": 808, "verb": "push", "text": "Continue pushing the keyboard into the designated frame area.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 7, "start": 808, "end": 838, "verb": "release", "text": "Release the keyboard and retract the arm.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 123, "t0": 0.0, "t1": 4.92, "l1_refs": { "left": [ 0, 1 ], "right": [] }, "text": "Place the alarm clock on the drawer.", "verbs": [ "grasp", "place" ], "objects": [ "alarm clock" ], "arms": [ "left" ], "flags": [] }, { "idx": 1, "start": 123, "end": 402, "t0": 4.92, "t1": 16.08, "l1_refs": { "left": [ 2, 3, 4 ], "right": [ 0 ] }, "text": "Place the figurine on the stand.", "verbs": [ "grasp", "place", "retreat" ], "objects": [ "figurine" ], "arms": [ "left" ], "flags": [] }, { "idx": 2, "start": 402, "end": 492, "t0": 16.08, "t1": 19.68, "l1_refs": { "left": [ 5 ], "right": [ 1, 2 ] }, "text": "Place the mouse on the mouse pad.", "verbs": [ "grasp", "place" ], "objects": [ "mouse", "keyboard" ], "arms": [ "left", "right" ], "flags": [ "splits_l1" ] }, { "idx": 3, "start": 492, "end": 838, "t0": 19.68, "t1": 33.52, "l1_refs": { "left": [ 6, 7, 8 ], "right": [ 3, 4, 5, 6, 7 ] }, "text": "Push the keyboard into the frame.", "verbs": [ "grasp", "push", "release", "hold" ], "objects": [ "keyboard" ], "arms": [ "left", "right" ], "flags": [ "splits_l1" ] } ], "duration_s": 33.52 }, { "id": "case07", "tree": "curobo_lerobot_v3/curobo_robodojo_v0", "episode": 628, "fps": 25, "n_frames": 612, "l3": "\"Pour the liquid from the first red bottle into the first black bowl, from the second turquoise bottle into the second brown bowl, and from the third violet bottle into the third white bowl. Then reset the robot arm.\"", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-031.mp4", "source_video_start_s": 70.88, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 33, "verb": "grasp", "text": "Grasp the red bottle.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 408, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 33, "end": 103, "verb": "lift", "text": "Lift the red bottle off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 103, "end": 159, "verb": "pour", "text": "Pour liquid from the red bottle into the black bowl.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 159, "end": 188, "verb": "place", "text": "Place the red bottle back on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 188, "end": 235, "verb": "grasp", "text": "Grasp the turquoise bottle.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 235, "end": 294, "verb": "lift", "text": "Lift the turquoise bottle off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 294, "end": 366, "verb": "pour", "text": "Pour liquid from the turquoise bottle into the brown bowl.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 7, "start": 366, "end": 395, "verb": "place", "text": "Place the turquoise bottle back on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 8, "start": 395, "end": 427, "verb": "retreat", "text": "Move the left arm away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 408, "end": 445, "verb": "grasp", "text": "Grasp the violet bottle.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 9, "start": 427, "end": 612, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 445, "end": 499, "verb": "lift", "text": "Lift the violet bottle off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [ "m_lift_noz" ], "mask_default": false, "mask_strict": true }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 499, "end": 552, "verb": "pour", "text": "Pour liquid from the violet bottle into the white bowl.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 552, "end": 583, "verb": "place", "text": "Place the violet bottle back on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 583, "end": 612, "verb": "retreat", "text": "Move the right arm backward away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 188, "t0": 0.0, "t1": 7.52, "l1_refs": { "left": [ 0, 1, 2, 3 ], "right": [] }, "text": "Pour liquid from the red bottle into the black bowl and place the red bottle back on the table.", "verbs": [ "grasp", "lift", "pour", "place" ], "objects": [ "bottle" ], "arms": [ "left" ], "flags": [] }, { "idx": 1, "start": 188, "end": 395, "t0": 7.52, "t1": 15.8, "l1_refs": { "left": [ 4, 5, 6, 7 ], "right": [ 0 ] }, "text": "Pour liquid from the turquoise bottle into the brown bowl and place the turquoise bottle back on the table.", "verbs": [ "grasp", "lift", "pour", "place" ], "objects": [ "bottle" ], "arms": [ "left" ], "flags": [] }, { "idx": 2, "start": 395, "end": 583, "t0": 15.8, "t1": 23.32, "l1_refs": { "left": [ 8, 9 ], "right": [ 1, 2, 3, 4 ] }, "text": "Pour liquid from the violet bottle into the white bowl and place the violet bottle back on the table.", "verbs": [ "retreat", "grasp", "lift", "pour", "place" ], "objects": [ "bottle" ], "arms": [ "right" ], "flags": [] }, { "idx": 3, "start": 583, "end": 612, "t0": 23.32, "t1": 24.48, "l1_refs": { "left": [], "right": [ 5 ] }, "text": "Retreat the right arm away from the workspace.", "verbs": [ "retreat" ], "objects": [], "arms": [], "flags": [ "passive_only" ] } ], "duration_s": 24.48 }, { "id": "case08", "tree": "curobo_lerobot_v3/curobo_robodojo_v0", "episode": 610, "fps": 25, "n_frames": 609, "l3": "\"Pour the liquid from the first red bottle into the first black bowl, from the second turquoise bottle into the second white bowl, and from the third violet bottle into the third brown bowl. Then reset the robot arm.\"", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-029.mp4", "source_video_start_s": 268.11999999999995, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 197, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 33, "verb": "grasp", "text": "Grasp the red bottle.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 33, "end": 108, "verb": "lift", "text": "Lift the red bottle off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 108, "end": 156, "verb": "pour", "text": "Pour liquid from the red bottle into the black bowl.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 156, "end": 184, "verb": "place", "text": "Place the red bottle upright on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 184, "end": 217, "verb": "retreat", "text": "Move the arm away from the bottles.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 197, "end": 234, "verb": "grasp", "text": "Grasp the turquoise bottle.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 217, "end": 385, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 234, "end": 288, "verb": "lift", "text": "Lift the turquoise bottle off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [ "m_lift_noz" ], "mask_default": false, "mask_strict": true }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 288, "end": 340, "verb": "pour", "text": "Pour liquid from the turquoise bottle into the white bowl.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 340, "end": 372, "verb": "place", "text": "Place the turquoise bottle back on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 372, "end": 404, "verb": "retreat", "text": "Retreat the left arm away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 6, "start": 385, "end": 422, "verb": "grasp", "text": "Grasp the violet bottle.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 404, "end": 609, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 7, "start": 422, "end": 480, "verb": "pour", "text": "Pour liquid from the violet bottle into the brown bowl.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 8, "start": 480, "end": 551, "verb": "place", "text": "Place the violet bottle upright on the table.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "right", "idx": 9, "start": 551, "end": 579, "verb": "place", "text": "Place the violet bottle upright on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 10, "start": 579, "end": 609, "verb": "retreat", "text": "Move the right arm backward away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 217, "t0": 0.0, "t1": 8.68, "l1_refs": { "left": [ 0, 1 ], "right": [ 0, 1, 2, 3, 4 ] }, "text": "Pour the liquid from the red bottle into the black bowl.", "verbs": [ "grasp", "lift", "pour", "place", "retreat" ], "objects": [ "bottle" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 1, "start": 217, "end": 404, "t0": 8.68, "t1": 16.16, "l1_refs": { "left": [ 2, 3, 4, 5 ], "right": [ 5, 6 ] }, "text": "Pour the liquid from the turquoise bottle into the white bowl.", "verbs": [ "lift", "pour", "place", "retreat", "grasp" ], "objects": [ "bottle" ], "arms": [ "left", "right" ], "flags": [ "splits_l1" ] }, { "idx": 2, "start": 404, "end": 609, "t0": 16.16, "t1": 24.36, "l1_refs": { "left": [ 6 ], "right": [ 7, 8, 9, 10 ] }, "text": "Pour the liquid from the violet bottle into the brown bowl and reset the arm.", "verbs": [ "pour", "place", "retreat" ], "objects": [ "bottle" ], "arms": [ "right" ], "flags": [ "splits_l1" ] } ], "duration_s": 24.36 }, { "id": "case09", "tree": "curobo_lerobot_v3/curobo_robodojo_v0_debug", "episode": 0, "fps": 25, "n_frames": 1536, "l3": "\"Insert and tighten each screw into the nut of the same color.\"", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-000.mp4", "source_video_start_s": 0.0, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 61, "verb": "grasp", "text": "Grasp the red screw on the left side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 424, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 61, "end": 129, "verb": "transport", "text": "Move the held red screw towards the blue nut.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 129, "end": 152, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 152, "end": 363, "verb": "rotate", "text": "Rotate the wrist to tighten the red screw into the blue nut.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 363, "end": 383, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 383, "end": 597, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 424, "end": 493, "verb": "grasp", "text": "Grasp the blue nut on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 493, "end": 560, "verb": "place", "text": "Place the blue nut onto the table surface.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 560, "end": 607, "verb": "retreat", "text": "Retreat the arm away from the placed nut.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 597, "end": 665, "verb": "grasp", "text": "Grasp the blue screw located in the center of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 607, "end": 1036, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 7, "start": 665, "end": 740, "verb": "transport", "text": "Move the held blue screw towards the red nut.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 8, "start": 740, "end": 764, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 9, "start": 764, "end": 975, "verb": "rotate", "text": "Rotate the wrist to tighten the blue screw into the red nut.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 10, "start": 975, "end": 995, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 11, "start": 995, "end": 1468, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 1036, "end": 1105, "verb": "grasp", "text": "Grasp the pink nut on the right side of the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 6, "start": 1105, "end": 1173, "verb": "lift", "text": "Lift the grasped pink nut off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [ "m_lift_noz" ], "mask_default": false, "mask_strict": true }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 7, "start": 1173, "end": 1197, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 8, "start": 1197, "end": 1408, "verb": "release", "text": "Release the pink nut onto the table.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": "release，但夹爪已张开", "upstream_compat": null, "signals": [ "s_g4_release_open" ], "mask_default": true, "mask_strict": true }, "compat": false, "grip_flag": "release，但夹爪已张开" }, { "arm": "right", "idx": 9, "start": 1408, "end": 1428, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 10, "start": 1428, "end": 1494, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 12, "start": 1468, "end": 1502, "verb": "retreat", "text": "Retreat the arm away from the assembled screws and nuts.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 11, "start": 1494, "end": 1536, "verb": "retreat", "text": "Move the arm backward and down to a resting position.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 13, "start": 1502, "end": 1536, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 383, "t0": 0.0, "t1": 15.32, "l1_refs": { "left": [ 0, 1, 2, 3, 4 ], "right": [ 0 ] }, "text": "Insert and tighten the red screw into the blue nut.", "verbs": [ "grasp", "transport", "rotate" ], "objects": [ "red screw" ], "arms": [ "left" ], "flags": [] }, { "idx": 1, "start": 383, "end": 607, "t0": 15.32, "t1": 24.28, "l1_refs": { "left": [ 5 ], "right": [ 1, 2, 3 ] }, "text": "Grasp and place the blue nut on the table surface.", "verbs": [ "grasp", "place", "retreat" ], "objects": [ "nut" ], "arms": [ "right" ], "flags": [] }, { "idx": 2, "start": 607, "end": 995, "t0": 24.28, "t1": 39.8, "l1_refs": { "left": [ 6, 7, 8, 9, 10 ], "right": [ 4 ] }, "text": "Insert and tighten the blue screw into the red nut.", "verbs": [ "grasp", "transport", "rotate" ], "objects": [ "blue screw" ], "arms": [ "left" ], "flags": [ "splits_l1" ] }, { "idx": 3, "start": 995, "end": 1428, "t0": 39.8, "t1": 57.12, "l1_refs": { "left": [ 11 ], "right": [ 5, 6, 7, 8, 9 ] }, "text": "Grasp and place the pink nut on the table surface.", "verbs": [ "grasp", "lift", "release" ], "objects": [ "nut" ], "arms": [ "right" ], "flags": [] }, { "idx": 4, "start": 1428, "end": 1536, "t0": 57.12, "t1": 61.44, "l1_refs": { "left": [ 12, 13 ], "right": [ 10, 11 ] }, "text": "Retreat both arms to a resting position.", "verbs": [ "retreat" ], "objects": [], "arms": [], "flags": [ "passive_only" ] } ], "duration_s": 61.44 }, { "id": "case10", "tree": "curobo_lerobot_v3/curobo_flat_eeaction", "episode": 52, "fps": 25, "n_frames": 367, "l3": "stack the blocks from bottom to top in the order ofblue, cyan, and green, then reset the robot arm.", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-001.mp4", "source_video_start_s": 157.60000000000002, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 19, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 19, "end": 39, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 39, "end": 90, "verb": "reach", "text": "Move the open gripper towards the blue block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 90, "end": 161, "verb": "grasp", "text": "Grasp the cyan block from above.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 161, "end": 205, "verb": "place", "text": "Place the cyan block on top of the blue block.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 5, "start": 205, "end": 266, "verb": "grasp", "text": "Grasp the green block from above.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 6, "start": 266, "end": 327, "verb": "place", "text": "Place the green block on top of the cyan block to complete the stack.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 7, "start": 327, "end": 367, "verb": "retreat", "text": "Retreat the left arm away from the stacked blocks.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 39, "t0": 0.0, "t1": 1.56, "l1_refs": { "left": [ 0, 1 ], "right": [] }, "text": "Reset the robot arm to its initial position.", "verbs": [], "objects": [], "arms": [], "flags": [ "passive_only" ] }, { "idx": 1, "start": 39, "end": 90, "t0": 1.56, "t1": 3.6, "l1_refs": { "left": [ 2 ], "right": [] }, "text": "Move the left gripper towards the blue block on the table.", "verbs": [ "reach" ], "objects": [ "block" ], "arms": [], "flags": [ "passive_only" ] }, { "idx": 2, "start": 90, "end": 205, "t0": 3.6, "t1": 8.2, "l1_refs": { "left": [ 3, 4 ], "right": [] }, "text": "Grasp the cyan block and place it on top of the blue block.", "verbs": [ "grasp", "place" ], "objects": [ "block" ], "arms": [ "left" ], "flags": [] }, { "idx": 3, "start": 205, "end": 327, "t0": 8.2, "t1": 13.08, "l1_refs": { "left": [ 5, 6 ], "right": [] }, "text": "Grasp the green block and place it on top of the cyan block to complete the stack.", "verbs": [ "grasp", "place" ], "objects": [ "block" ], "arms": [ "left" ], "flags": [] }, { "idx": 4, "start": 327, "end": 367, "t0": 13.08, "t1": 14.68, "l1_refs": { "left": [ 7 ], "right": [] }, "text": "Retreat the left arm away from the stacked blocks.", "verbs": [ "retreat" ], "objects": [], "arms": [], "flags": [ "passive_only" ] } ], "duration_s": 14.68 }, { "id": "case11", "tree": "curobo_lerobot_v3/curobo_flat_eeaction", "episode": 104, "fps": 25, "n_frames": 364, "l3": "stack the blocks from bottom to top in the order ofblue, cyan, and yellow, then reset the robot arm.", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-002.mp4", "source_video_start_s": 317.28000000000003, "l1": [ { "arm": "right", "idx": 0, "start": 0, "end": 18, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 18, "end": 32, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 32, "end": 86, "verb": "place", "text": "Place the cyan block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 86, "end": 136, "verb": "grasp", "text": "Grasp the blue block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 136, "end": 150, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 5, "start": 150, "end": 201, "verb": "place", "text": "Place the blue block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 6, "start": 201, "end": 263, "verb": "grasp", "text": "Grasp the yellow block on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 7, "start": 263, "end": 323, "verb": "place", "text": "Place the yellow block on top of the blue block.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 8, "start": 323, "end": 364, "verb": "retreat", "text": "Move the right arm away from the stacked blocks.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 86, "t0": 0.0, "t1": 3.44, "l1_refs": { "left": [], "right": [ 0, 1, 2 ] }, "text": "Place the cyan block on the table.", "verbs": [ "place" ], "objects": [ "block" ], "arms": [ "right" ], "flags": [] }, { "idx": 1, "start": 86, "end": 201, "t0": 3.44, "t1": 8.04, "l1_refs": { "left": [], "right": [ 3, 4, 5 ] }, "text": "Place the blue block on the table.", "verbs": [ "grasp", "place" ], "objects": [ "block" ], "arms": [ "right" ], "flags": [] }, { "idx": 2, "start": 201, "end": 323, "t0": 8.04, "t1": 12.92, "l1_refs": { "left": [], "right": [ 6, 7 ] }, "text": "Place the yellow block on top of the blue block.", "verbs": [ "grasp", "place" ], "objects": [ "block" ], "arms": [ "right" ], "flags": [] }, { "idx": 3, "start": 323, "end": 364, "t0": 12.92, "t1": 14.56, "l1_refs": { "left": [], "right": [ 8 ] }, "text": "Move the right arm away from the stacked blocks.", "verbs": [ "retreat" ], "objects": [], "arms": [], "flags": [ "passive_only" ] } ], "duration_s": 14.56 }, { "id": "case12", "tree": "curobo_lerobot_v3/curobo_flat_scene", "episode": 0, "fps": 25, "n_frames": 213, "l3": "Pick up the coin from the holder and insert it precisely into the coin bank.", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-000.mp4", "source_video_start_s": 0.0, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 33, "verb": "reach", "text": "Move the left arm towards the coin holder on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 81, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 33, "end": 51, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 51, "end": 129, "verb": "handover", "text": "Hand over the coin to the right arm.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 81, "end": 129, "verb": "grasp", "text": "Move the right arm to the coin held by the left arm and grasp it.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 129, "end": 166, "verb": "retreat", "text": "Retreat the left arm away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 129, "end": 192, "verb": "insert", "text": "Insert the coin into the slot of the pink piggy bank.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": null }, { "arm": "left", "idx": 4, "start": 166, "end": 213, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 192, "end": 213, "verb": null, "text": null, "not_sent": "short", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 51, "t0": 0.0, "t1": 2.04, "l1_refs": { "left": [ 0, 1 ], "right": [ 0 ] }, "text": "Pick up the coin from the holder with the left arm.", "verbs": [ "reach" ], "objects": [ "coin holder" ], "arms": [], "flags": [ "passive_only" ] }, { "idx": 1, "start": 51, "end": 129, "t0": 2.04, "t1": 5.16, "l1_refs": { "left": [ 2 ], "right": [ 1 ] }, "text": "Hand over the coin from the left gripper to the right gripper.", "verbs": [ "handover", "grasp" ], "objects": [ "coin" ], "arms": [ "left", "right" ], "flags": [] }, { "idx": 2, "start": 129, "end": 213, "t0": 5.16, "t1": 8.52, "l1_refs": { "left": [ 3, 4 ], "right": [ 2, 3 ] }, "text": "Insert the coin into the slot of the pink piggy bank with the right arm.", "verbs": [ "retreat", "insert" ], "objects": [ "coin" ], "arms": [ "right" ], "flags": [] } ], "duration_s": 8.52 }, { "id": "case13", "tree": "curobo_lerobot_v3/curobo_flat", "episode": 0, "fps": 25, "n_frames": 215, "l3": "Pick up the key, hand it over to the other hand, insert it into the keyhole, then turn it.", "camera": "observation.images.cam_head", "source_video": "videos/observation.images.cam_head/chunk-000/file-000.mp4", "source_video_start_s": 0.0, "l1": [ { "arm": "left", "idx": 0, "start": 0, "end": 62, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null }, { "arm": "right", "idx": 0, "start": 0, "end": 36, "verb": "grasp", "text": "Grasp the key on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 1, "start": 36, "end": 66, "verb": "lift", "text": "Lift the key off the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 1, "start": 62, "end": 116, "verb": "grasp", "text": "Grasp the key on the table.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 2, "start": 66, "end": 95, "verb": "hold", "text": "Hold the key steady for the left arm to grasp.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 3, "start": 95, "end": 123, "verb": "release", "text": "Release the key into the left gripper.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 2, "start": 116, "end": 146, "verb": "hold", "text": "Hold the key steady for the right arm to take.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "right", "idx": 4, "start": 123, "end": 153, "verb": "retreat", "text": "Retreat the right arm away from the workspace.", "not_sent": null, "qc": { "rule_compat": true, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": true, "grip_flag": null }, { "arm": "left", "idx": 3, "start": 146, "end": 215, "verb": "retreat", "text": "Retreat the left arm away from the workspace.", "not_sent": null, "qc": { "rule_compat": false, "grip_flag": "空夹爪动词，夹爪闭合", "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": false, "grip_flag": "空夹爪动词，夹爪闭合" }, { "arm": "right", "idx": 5, "start": 153, "end": 215, "verb": null, "text": null, "not_sent": "idle_open", "qc": { "rule_compat": null, "grip_flag": null, "upstream_compat": null, "signals": [], "mask_default": false, "mask_strict": false }, "compat": null, "grip_flag": null } ], "l2": [ { "idx": 0, "start": 0, "end": 62, "t0": 0.0, "t1": 2.48, "l1_refs": { "left": [ 0 ], "right": [ 0, 1 ] }, "text": "Pick up the key with the right arm.", "verbs": [ "grasp", "lift" ], "objects": [ "key" ], "arms": [ "right" ], "flags": [] }, { "idx": 1, "start": 62, "end": 123, "t0": 2.48, "t1": 4.92, "l1_refs": { "left": [ 1 ], "right": [ 2, 3 ] }, "text": "Hand the key from the right gripper to the left gripper.", "verbs": [ "grasp", "hold", "release" ], "objects": [ "key" ], "arms": [ "left", "right" ], "flags": [ "splits_l1" ] }, { "idx": 2, "start": 123, "end": 215, "t0": 4.92, "t1": 8.6, "l1_refs": { "left": [ 2, 3 ], "right": [ 4, 5 ] }, "text": "Retreat both arms away from the workspace.", "verbs": [ "hold", "retreat" ], "objects": [ "key" ], "arms": [], "flags": [ "splits_l1", "passive_only" ] } ], "duration_s": 8.6 } ]