dlc2action.data.annotation_store
Specific implementations of dlc2action.data.base_store.AnnotationStore
are defined here
1# 2# Copyright 2020-2022 by A. Mathis Group and contributors. All rights reserved. 3# 4# This project and all its files are licensed under GNU AGPLv3 or later version. A copy is included in dlc2action/LICENSE.AGPL. 5# 6""" 7Specific implementations of `dlc2action.data.base_store.AnnotationStore` are defined here 8""" 9 10from dlc2action.data.base_store import AnnotationStore 11from dlc2action.utils import strip_suffix 12from typing import Dict, List, Tuple, Set, Union 13import torch 14import numpy as np 15import pickle 16from copy import copy 17from collections import Counter 18from collections.abc import Iterable 19from abc import abstractmethod 20import os 21from tqdm import tqdm 22import pandas as pd 23from collections import defaultdict 24from itertools import combinations 25 26 27class EmptyAnnotationStore(AnnotationStore): 28 def __init__( 29 self, video_order: List = None, key_objects: Tuple = None, *args, **kwargs 30 ): 31 """ 32 Parameters 33 ---------- 34 video_order : list, optional 35 a list of video ids that should be processed in the same order (not passed if creating from key objects) 36 key_objects : tuple, optional 37 a tuple of key objects 38 """ 39 40 pass 41 42 def __len__(self) -> int: 43 """ 44 Get the number of available samples 45 46 Returns 47 ------- 48 length : int 49 the number of available samples 50 """ 51 52 return 0 53 54 def remove(self, indices: List) -> None: 55 """ 56 Remove the samples corresponding to indices 57 58 Parameters 59 ---------- 60 indices : int 61 a list of integer indices to remove 62 """ 63 64 pass 65 66 def key_objects(self) -> Tuple: 67 """ 68 Return a tuple of the key objects necessary to re-create the Store 69 70 Returns 71 ------- 72 key_objects : tuple 73 a tuple of key objects 74 """ 75 76 return () 77 78 def load_from_key_objects(self, key_objects: Tuple) -> None: 79 """ 80 Load the information from a tuple of key objects 81 82 Parameters 83 ---------- 84 key_objects : tuple 85 a tuple of key objects 86 """ 87 88 pass 89 90 def to_ram(self) -> None: 91 """ 92 Transfer the data samples to RAM if they were previously stored as file paths 93 """ 94 95 pass 96 97 def get_original_coordinates(self) -> np.ndarray: 98 """ 99 Return the original coordinates array 100 101 Returns 102 ------- 103 np.ndarray 104 an array that contains the coordinates of the data samples in original input data (video id, clip id, 105 start frame) 106 """ 107 108 return None 109 110 def create_subsample(self, indices: List, ssl_indices: List = None): 111 """ 112 Create a new store that contains a subsample of the data 113 114 Parameters 115 ---------- 116 indices : list 117 the indices to be included in the subsample 118 ssl_indices : list, optional 119 the indices to be included in the subsample without the annotation data 120 """ 121 122 return self.new() 123 124 @classmethod 125 def get_file_ids(cls, *args, **kwargs) -> List: 126 """ 127 Process data parameters and return a list of ids of the videos that should 128 be processed by the __init__ function 129 130 Returns 131 ------- 132 video_ids : list 133 a list of video file ids 134 """ 135 136 return None 137 138 def __getitem__(self, ind: int) -> torch.Tensor: 139 """ 140 Return the annotation of the sample corresponding to an index 141 142 Parameters 143 ---------- 144 ind : int 145 index of the sample 146 147 Returns 148 ------- 149 sample : torch.Tensor 150 the corresponding annotation tensor 151 """ 152 153 return torch.tensor(float("nan")) 154 155 def get_len(self, return_unlabeled: bool) -> int: 156 """ 157 Get the length of the subsample of labeled/unlabeled data 158 159 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 160 and if return_unlabeled is None the index is already correct 161 162 Parameters 163 ---------- 164 return_unlabeled : bool 165 the identifier for the subsample 166 167 Returns 168 ------- 169 length : int 170 the length of the subsample 171 """ 172 173 return None 174 175 def get_idx(self, index: int, return_unlabeled: bool) -> int: 176 """ 177 Convert from an index in the subsample of labeled/unlabeled data to an index in the full array 178 179 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 180 and if return_unlabeled is None the index is already correct 181 182 Parameters 183 ---------- 184 index : int 185 the index in the subsample 186 return_unlabeled : bool 187 the identifier for the subsample 188 189 Returns 190 ------- 191 corrected_index : int 192 the index in the full dataset 193 """ 194 195 return index 196 197 def count_classes( 198 self, frac: bool = False, zeros: bool = False, bouts: bool = False 199 ) -> Dict: 200 """ 201 Get a dictionary with class-wise frame counts 202 203 Parameters 204 ---------- 205 frac : bool, default False 206 if True, a fraction of the total frame count is returned 207 208 Returns 209 ------- 210 count_dictionary : dict 211 a dictionary with class indices as keys and frame counts as values 212 """ 213 214 return {} 215 216 def behaviors_dict(self) -> Dict: 217 """ 218 Get a dictionary of class names 219 220 Returns 221 ------- 222 behavior_dictionary: dict 223 a dictionary with class indices as keys and class names as values 224 """ 225 226 return {} 227 228 def annotation_class(self) -> str: 229 """ 230 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon) 231 232 Returns 233 ------- 234 annotation_class : str 235 the type of annotation 236 """ 237 238 return "none" 239 240 def size(self) -> int: 241 """ 242 Get the total number of frames in the data 243 244 Returns 245 ------- 246 size : int 247 the total number of frames 248 """ 249 250 return None 251 252 def filtered_indices(self) -> List: 253 """ 254 Return the indices of the samples that should be removed 255 256 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 257 parameters 258 259 Returns 260 ------- 261 indices_to_remove : list 262 a list of integer indices that should be removed 263 """ 264 265 return [] 266 267 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 268 """ 269 Set pseudo labels to the unlabeled data 270 271 Parameters 272 ---------- 273 labels : torch.Tensor 274 a tensor of pseudo-labels for the unlabeled data 275 """ 276 277 pass 278 279 280class ActionSegmentationStore(AnnotationStore): # + 281 """ 282 A general realization of an annotation store for action segmentation tasks 283 284 Assumes the following file structure: 285 ``` 286 annotation_path 287 ├── video1_annotation.pickle 288 └── video2_labels.pickle 289 ``` 290 Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`. 291 """ 292 293 def __init__( 294 self, 295 video_order: List = None, 296 min_frames: Dict = None, 297 max_frames: Dict = None, 298 visibility: Dict = None, 299 exclusive: bool = True, 300 len_segment: int = 128, 301 overlap: int = 0, 302 behaviors: Set = None, 303 ignored_classes: Set = None, 304 ignored_clips: Set = None, 305 annotation_suffix: Union[Set, str] = None, 306 annotation_path: Union[Set, str] = None, 307 behavior_file: str = None, 308 correction: Dict = None, 309 frame_limit: int = 0, 310 filter_annotated: bool = False, 311 filter_background: bool = False, 312 error_class: str = None, 313 min_frames_action: int = None, 314 key_objects: Tuple = None, 315 visibility_min_score: float = 0.2, 316 visibility_min_frac: float = 0.7, 317 mask: Dict = None, 318 use_hard_negatives: bool = False, 319 interactive: bool = False, 320 *args, 321 **kwargs, 322 ) -> None: 323 """ 324 Parameters 325 ---------- 326 video_order : list, optional 327 a list of video ids that should be processed in the same order (not passed if creating from key objects) 328 min_frames : dict, optional 329 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 330 clip start frames (not passed if creating from key objects) 331 max_frames : dict, optional 332 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 333 clip end frames (not passed if creating from key objects) 334 visibility : dict, optional 335 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 336 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 337 exclusive : bool, default True 338 if True, the annotation is single-label; if False, multi-label 339 len_segment : int, default 128 340 the length of the segments in which the data should be cut (in frames) 341 overlap : int, default 0 342 the length of the overlap between neighboring segments (in frames) 343 behaviors : set, optional 344 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 345 loaded from a file) 346 ignored_classes : set, optional 347 the list of behaviors from the behaviors list or file to not annotate 348 ignored_clips : set, optional 349 clip ids to ignore 350 annotation_suffix : str | set, optional 351 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 352 (not passed if creating from key objects or if irrelevant for the dataset) 353 annotation_path : str | set, optional 354 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 355 from key objects) 356 behavior_file : str, optional 357 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 358 correction : dict, optional 359 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 360 can be used to correct for variations in naming or to merge several labels in one 361 frame_limit : int, default 0 362 the smallest possible length of a clip (shorter clips are discarded) 363 filter_annotated : bool, default False 364 if True, the samples that do not have any labels will be filtered 365 filter_background : bool, default False 366 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 367 error_class : str, optional 368 the name of the error class (the annotations that intersect with this label will be discarded) 369 min_frames_action : int, default 0 370 the minimum length of an action (shorter actions are not annotated) 371 key_objects : tuple, optional 372 the key objects to load the AnnotationStore from 373 visibility_min_score : float, default 5 374 the minimum visibility score for visibility filtering 375 visibility_min_frac : float, default 0.7 376 the minimum fraction of visible frames for visibility filtering 377 mask : dict, optional 378 a masked value dictionary (for active learning simulation experiments) 379 use_hard_negatives : bool, default False 380 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 381 interactive : bool, default False 382 if `True`, annotation is assigned to pairs of individuals 383 """ 384 385 super().__init__() 386 387 if ignored_clips is None: 388 ignored_clips = [] 389 self.len_segment = int(len_segment) 390 self.exclusive = exclusive 391 if overlap < 1: 392 overlap = overlap * len_segment 393 self.overlap = int(overlap) 394 self.video_order = video_order 395 self.min_frames = min_frames 396 self.max_frames = max_frames 397 self.visibility = visibility 398 self.vis_min_score = visibility_min_score 399 self.vis_min_frac = visibility_min_frac 400 self.mask = mask 401 self.use_negatives = use_hard_negatives 402 self.interactive = interactive 403 self.ignored_clips = ignored_clips 404 self.file_paths = self._get_file_paths(annotation_path) 405 self.ignored_classes = ignored_classes 406 self.update_behaviors = False 407 408 self.ram = True 409 self.original_coordinates = [] 410 self.filtered = [] 411 412 self.step = self.len_segment - self.overlap 413 414 self.ann_suffix = annotation_suffix 415 self.annotation_folder = annotation_path 416 self.filter_annotated = filter_annotated 417 self.filter_background = filter_background 418 self.frame_limit = frame_limit 419 self.min_frames_action = min_frames_action 420 self.error_class = error_class 421 422 if correction is None: 423 correction = {} 424 self.correction = correction 425 426 if self.max_frames is None: 427 self.max_frames = defaultdict(lambda: {}) 428 if self.min_frames is None: 429 self.min_frames = defaultdict(lambda: {}) 430 431 lists = [self.annotation_folder, self.ann_suffix] 432 for i in range(len(lists)): 433 iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str)) 434 if lists[i] is not None: 435 if not iterable: 436 lists[i] = [lists[i]] 437 lists[i] = [x for x in lists[i]] 438 self.annotation_folder, self.ann_suffix = lists 439 440 if ignored_classes is None: 441 ignored_classes = [] 442 self.ignored_classes = ignored_classes 443 self._set_behaviors(behaviors, ignored_classes, behavior_file) 444 445 if key_objects is None and self.video_order is not None: 446 self.data = self._load_data() 447 elif key_objects is not None: 448 self.load_from_key_objects(key_objects) 449 else: 450 self.data = None 451 self.labeled_indices, self.unlabeled_indices = self._compute_labeled() 452 453 def __getitem__(self, ind): 454 if self.data is None: 455 raise RuntimeError("The annotation store data has not been initialized!") 456 return self.data[ind] 457 458 def __len__(self) -> int: 459 if self.data is None: 460 raise RuntimeError("The annotation store data has not been initialized!") 461 return len(self.data) 462 463 def remove(self, indices: List) -> None: 464 """ 465 Remove the samples corresponding to indices 466 467 Parameters 468 ---------- 469 indices : list 470 a list of integer indices to remove 471 """ 472 473 if len(indices) > 0: 474 mask = np.ones(len(self.data)) 475 mask[indices] = 0 476 mask = mask.astype(bool) 477 self.data = self.data[mask] 478 self.original_coordinates = self.original_coordinates[mask] 479 480 def key_objects(self) -> Tuple: 481 """ 482 Return a tuple of the key objects necessary to re-create the Store 483 484 Returns 485 ------- 486 key_objects : tuple 487 a tuple of key objects 488 """ 489 490 return ( 491 self.original_coordinates, 492 self.data, 493 self.behaviors, 494 self.exclusive, 495 self.len_segment, 496 self.step, 497 self.overlap, 498 ) 499 500 def load_from_key_objects(self, key_objects: Tuple) -> None: 501 """ 502 Load the information from a tuple of key objects 503 504 Parameters 505 ---------- 506 key_objects : tuple 507 a tuple of key objects 508 """ 509 510 ( 511 self.original_coordinates, 512 self.data, 513 self.behaviors, 514 self.exclusive, 515 self.len_segment, 516 self.step, 517 self.overlap, 518 ) = key_objects 519 self.labeled_indices, self.unlabeled_indices = self._compute_labeled() 520 521 def to_ram(self) -> None: 522 """ 523 Transfer the data samples to RAM if they were previously stored as file paths 524 """ 525 526 pass 527 528 def get_original_coordinates(self) -> np.ndarray: 529 """ 530 Return the video_indices array 531 532 Returns 533 ------- 534 original_coordinates : numpy.ndarray 535 an array that contains the coordinates of the data samples in original input data 536 """ 537 return self.original_coordinates 538 539 def create_subsample(self, indices: List, ssl_indices: List = None): 540 """ 541 Create a new store that contains a subsample of the data 542 543 Parameters 544 ---------- 545 indices : list 546 the indices to be included in the subsample 547 ssl_indices : list, optional 548 the indices to be included in the subsample without the annotation data 549 """ 550 551 if ssl_indices is None: 552 ssl_indices = [] 553 data = copy(self.data) 554 data[ssl_indices, ...] = -100 555 new = self.new() 556 new.original_coordinates = self.original_coordinates[indices + ssl_indices] 557 new.data = self.data[indices + ssl_indices] 558 new.labeled_indices, new.unlabeled_indices = new._compute_labeled() 559 new.behaviors = self.behaviors 560 new.exclusive = self.exclusive 561 new.len_segment = self.len_segment 562 new.step = self.step 563 new.overlap = self.overlap 564 new.max_frames = self.max_frames 565 new.min_frames = self.min_frames 566 return new 567 568 def get_len(self, return_unlabeled: bool) -> int: 569 """ 570 Get the length of the subsample of labeled/unlabeled data 571 572 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 573 and if return_unlabeled is None the index is already correct 574 575 Parameters 576 ---------- 577 return_unlabeled : bool 578 the identifier for the subsample 579 580 Returns 581 ------- 582 length : int 583 the length of the subsample 584 """ 585 586 if self.data is None: 587 raise RuntimeError("The annotation store data has not been initialized!") 588 elif return_unlabeled is None: 589 return len(self.data) 590 elif return_unlabeled: 591 return len(self.unlabeled_indices) 592 else: 593 return len(self.labeled_indices) 594 595 def get_indices(self, return_unlabeled: bool) -> List: 596 """ 597 Get a list of indices of samples in the labeled/unlabeled subset 598 599 Parameters 600 ---------- 601 return_unlabeled : bool 602 the identifier for the subsample (`True` for unlabeled, `False` for labeled, `None` for the 603 whole dataset) 604 605 Returns 606 ------- 607 indices : list 608 a list of indices that meet the criteria 609 """ 610 611 return list(range(len(self.data))) 612 613 def count_classes( 614 self, perc: bool = False, zeros: bool = False, bouts: bool = False 615 ) -> Dict: 616 """ 617 Get a dictionary with class-wise frame counts 618 619 Parameters 620 ---------- 621 perc : bool, default False 622 if `True`, a fraction of the total frame count is returned 623 zeros : bool, default False 624 if `True` and annotation is not exclusive, zero counts are returned 625 bouts : bool, default False 626 if `True`, instead of frame counts segment counts are returned 627 628 Returns 629 ------- 630 count_dictionary : dict 631 a dictionary with class indices as keys and frame counts as values 632 """ 633 634 if bouts: 635 if self.overlap != 0: 636 data = {} 637 for video, value in self.max_frames.items(): 638 for clip, end in value.items(): 639 length = end - self._get_min_frame(video, clip) 640 if self.exclusive: 641 data[f"{video}---{clip}"] = -100 * torch.ones(length) 642 else: 643 data[f"{video}---{clip}"] = -100 * torch.ones( 644 (len(self.behaviors_dict()), length) 645 ) 646 for x, coords in zip(self.data, self.original_coordinates): 647 split = coords[0].split("---") 648 l = self._get_max_frame(split[0], split[1]) - self._get_min_frame( 649 split[0], split[1] 650 ) 651 i = coords[1] 652 start = int(i) * self.step 653 end = min(start + self.len_segment, l) 654 data[coords[0]][..., start:end] = x[..., : end - start] 655 values = [] 656 for key, value in data.items(): 657 values.append(value) 658 values.append(-100 * torch.ones((*value.shape[:-1], 1))) 659 data = torch.cat(values, -1).T 660 else: 661 data = copy(self.data) 662 if self.exclusive: 663 data = data.flatten() 664 else: 665 data = data.transpose(1, 2).reshape(-1, len(self.behaviors)) 666 count_dictionary = {} 667 for c in self.behaviors_dict(): 668 if self.exclusive: 669 arr = data == c 670 else: 671 if zeros: 672 arr = data[:, c] == 0 673 else: 674 arr = data[:, c] == 1 675 output, indices = torch.unique_consecutive(arr, return_inverse=True) 676 true_indices = torch.where(output)[0] 677 count_dictionary[c] = len(true_indices) 678 else: 679 ind = 1 680 if zeros: 681 ind = 0 682 if self.exclusive: 683 count_dictionary = dict(Counter(self.data.flatten().cpu().numpy())) 684 else: 685 d = {} 686 for i in range(self.data.shape[1]): 687 cnt = Counter(self.data[:, i, :].flatten().cpu().numpy()) 688 d[i] = cnt[ind] 689 count_dictionary = d 690 if perc: 691 total = sum([v for k, v in count_dictionary.items()]) 692 count_dictionary = {k: v / total for k, v in count_dictionary.items()} 693 for i in self.behaviors_dict(): 694 if i not in count_dictionary: 695 count_dictionary[i] = 0 696 return count_dictionary 697 698 def behaviors_dict(self) -> Dict: 699 """ 700 Get a dictionary of class names 701 702 Returns 703 ------- 704 behavior_dictionary: dict 705 a dictionary with class indices as keys and class names as values 706 """ 707 708 # if self.behaviors is None 709 if self.exclusive and "other" not in self.behaviors: 710 d = {i + 1: b for i, b in enumerate(self.behaviors)} 711 d[0] = "other" 712 else: 713 d = {i: b for i, b in enumerate(self.behaviors)} 714 return d 715 716 def annotation_class(self) -> str: 717 """ 718 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification') 719 720 Returns 721 ------- 722 annotation_class : str 723 the type of annotation 724 """ 725 726 if self.exclusive: 727 return "exclusive_classification" 728 else: 729 return "nonexclusive_classification" 730 731 def size(self) -> int: 732 """ 733 Get the total number of frames in the data 734 735 Returns 736 ------- 737 size : int 738 the total number of frames 739 """ 740 741 return self.data.shape[0] * self.data.shape[-1] 742 743 def filtered_indices(self) -> List: 744 """ 745 Return the indices of the samples that should be removed 746 747 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 748 parameters 749 750 Returns 751 ------- 752 indices_to_remove : list 753 a list of integer indices that should be removed 754 """ 755 756 return self.filtered 757 758 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 759 """ 760 Set pseudo labels to the unlabeled data 761 762 Parameters 763 ---------- 764 labels : torch.Tensor 765 a tensor of pseudo-labels for the unlabeled data 766 """ 767 768 self.data[self.unlabeled_indices] = labels 769 770 @classmethod 771 def get_file_ids( 772 cls, 773 annotation_path: Union[str, Set], 774 annotation_suffix: Union[str, Set], 775 *args, 776 **kwargs, 777 ) -> List: 778 """ 779 Process data parameters and return a list of ids of the videos that should 780 be processed by the __init__ function 781 782 Parameters 783 ---------- 784 annotation_path : str | set 785 the path or the set of paths to the folder where the annotation files are stored 786 annotation_suffix : str | set, optional 787 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 788 789 Returns 790 ------- 791 video_ids : list 792 a list of video file ids 793 """ 794 795 lists = [annotation_path, annotation_suffix] 796 for i in range(len(lists)): 797 iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str)) 798 if lists[i] is not None: 799 if not iterable: 800 lists[i] = [lists[i]] 801 lists[i] = [x for x in lists[i]] 802 annotation_path, annotation_suffix = lists 803 files = [] 804 for folder in annotation_path: 805 files += [ 806 strip_suffix(os.path.basename(file), annotation_suffix) 807 for file in os.listdir(folder) 808 if file.endswith(tuple([x for x in annotation_suffix])) 809 ] 810 files = sorted(files, key=lambda x: os.path.basename(x)) 811 return files 812 813 def _set_behaviors( 814 self, behaviors: List, ignored_classes: List, behavior_file: str 815 ): 816 """ 817 Get a list of behaviors that should be annotated from behavior parameters 818 """ 819 820 if behaviors is not None: 821 for b in ignored_classes: 822 if b in behaviors: 823 behaviors.remove(b) 824 self.behaviors = behaviors 825 826 def _compute_labeled(self) -> Tuple[torch.Tensor, torch.Tensor]: 827 """ 828 Get the indices of labeled (annotated) and unlabeled samples 829 """ 830 831 if self.data is not None and len(self.data) > 0: 832 unlabeled = torch.sum(self.data != -100, dim=1) == 0 833 labeled_indices = torch.where(~unlabeled)[0] 834 unlabeled_indices = torch.where(unlabeled)[0] 835 else: 836 labeled_indices, unlabeled_indices = torch.tensor([]), torch.tensor([]) 837 return labeled_indices, unlabeled_indices 838 839 def _generate_annotation(self, times: Dict, name: str) -> Dict: 840 """ 841 Process a loaded annotation file to generate a training labels dictionary 842 """ 843 844 annotation = {} 845 if self.behaviors is None and times is not None: 846 behaviors = set() 847 for d in times.values(): 848 behaviors.update([k for k, v in d.items()]) 849 self.behaviors = [ 850 x 851 for x in sorted(behaviors) 852 if x not in self.ignored_classes 853 and not x.startswith("negative") 854 and not x.startswith("unknown") 855 ] 856 elif self.behaviors is None and times is None: 857 raise ValueError("Cannot generate annotqtion without behavior information!") 858 beh_inv = {v: k for k, v in self.behaviors_dict().items()} 859 # if there is no annotation file, generate empty annotation 860 if self.interactive: 861 clips = [ 862 "+".join(sorted(x)) 863 for x in combinations(self.max_frames[name].keys(), 2) 864 ] 865 else: 866 clips = list(self.max_frames[name].keys()) 867 if times is None: 868 clips = [x for x in clips if x not in self.ignored_clips] 869 # otherwise, apply filters and generate label arrays 870 else: 871 clips = [ 872 x 873 for x in clips 874 if x not in self.ignored_clips and x not in times.keys() 875 ] 876 for ind in times.keys(): 877 try: 878 min_frame = self._get_min_frame(name, ind) 879 max_frame = self._get_max_frame(name, ind) 880 except KeyError: 881 continue 882 go_on = max_frame - min_frame + 1 >= self.frame_limit 883 if go_on: 884 v_len = max_frame - min_frame + 1 885 if self.exclusive: 886 if not self.filter_background: 887 value = beh_inv.get("other", 0) 888 labels = np.ones(v_len, dtype=np.compat.long) * value 889 else: 890 labels = -100 * np.ones(v_len, dtype=np.compat.long) 891 else: 892 labels = np.zeros((len(self.behaviors), v_len), dtype=np.float) 893 cat_new = [] 894 for cat in times[ind].keys(): 895 if cat.startswith("unknown"): 896 cat_new.append(cat) 897 for cat in times[ind].keys(): 898 if cat.startswith("negative"): 899 cat_new.append(cat) 900 for cat in times[ind].keys(): 901 if not cat.startswith("negative") and not cat.startswith( 902 "unknown" 903 ): 904 cat_new.append(cat) 905 for cat in cat_new: 906 neg = False 907 unknown = False 908 cat_times = times[ind][cat] 909 if self.use_negatives and cat.startswith("negative"): 910 cat = " ".join(cat.split()[1:]) 911 neg = True 912 elif cat.startswith("unknown"): 913 cat = " ".join(cat.split()[1:]) 914 unknown = True 915 if cat in self.correction: 916 cat = self.correction[cat] 917 for start, end, amb in cat_times: 918 if end > self._get_max_frame(name, ind) + 1: 919 end = self._get_max_frame(name, ind) + 1 920 if amb != 0: 921 continue 922 start -= min_frame 923 end -= min_frame 924 if ( 925 self.min_frames_action is not None 926 and end - start < self.min_frames_action 927 ): 928 continue 929 if ( 930 self.vis_min_frac > 0 931 and self.vis_min_score > 0 932 and self.visibility is not None 933 ): 934 s = 0 935 for ind_k in ind.split("+"): 936 s += np.sum( 937 self.visibility[name][ind_k][start:end] 938 > self.vis_min_score 939 ) 940 if s < self.vis_min_frac * (end - start) * len( 941 ind.split("+") 942 ): 943 continue 944 if cat in beh_inv: 945 cat_i_global = beh_inv[cat] 946 if self.exclusive: 947 labels[start:end] = cat_i_global 948 else: 949 if unknown: 950 labels[cat_i_global, start:end] = -100 951 elif neg: 952 labels[cat_i_global, start:end] = 2 953 else: 954 labels[cat_i_global, start:end] = 1 955 else: 956 self.not_found.add(cat) 957 if self.filter_background: 958 if not self.exclusive: 959 labels[:, start:end][ 960 labels[:, start:end] == 0 961 ] = 3 962 else: 963 labels[start:end][labels[start:end] == -100] = 0 964 965 if self.error_class is not None and self.error_class in times[ind]: 966 for start, end, amb in times[ind][self.error_class]: 967 if self.exclusive: 968 labels[start:end] = -100 969 else: 970 labels[:, start:end] = -100 971 annotation[os.path.basename(name) + "---" + str(ind)] = labels 972 for ind in clips: 973 try: 974 min_frame = self._get_min_frame(name, ind) 975 max_frame = self._get_max_frame(name, ind) 976 except KeyError: 977 continue 978 go_on = max_frame - min_frame + 1 >= self.frame_limit 979 if go_on: 980 v_len = max_frame - min_frame + 1 981 if self.exclusive: 982 annotation[ 983 os.path.basename(name) + "---" + str(ind) 984 ] = -100 * np.ones(v_len, dtype=np.compat.long) 985 else: 986 annotation[ 987 os.path.basename(name) + "---" + str(ind) 988 ] = -100 * np.ones((len(self.behaviors), v_len), dtype=np.float) 989 return annotation 990 991 def _make_trimmed_annotations(self, annotations_dict: Dict) -> torch.Tensor: 992 """ 993 Cut a label dictionary into overlapping pieces of equal length 994 """ 995 996 labels = [] 997 self.original_coordinates = [] 998 masked_all = [] 999 for v_id in sorted(annotations_dict.keys()): 1000 if v_id in annotations_dict: 1001 annotations = annotations_dict[v_id] 1002 else: 1003 raise ValueError( 1004 f'The id list in {v_id.split("---")[0]} is not consistent across files' 1005 ) 1006 split = v_id.split("---") 1007 if len(split) > 1: 1008 video_id, ind = split 1009 else: 1010 video_id = split[0] 1011 ind = "" 1012 min_frame = self._get_min_frame(video_id, ind) 1013 max_frame = self._get_max_frame(video_id, ind) 1014 v_len = max_frame - min_frame + 1 1015 sp = np.arange(0, v_len, self.step) 1016 pad = sp[-1] + self.len_segment - v_len 1017 if self.exclusive: 1018 annotations = np.pad(annotations, ((0, pad)), constant_values=-100) 1019 else: 1020 annotations = np.pad( 1021 annotations, ((0, 0), (0, pad)), constant_values=-100 1022 ) 1023 masked = np.zeros(annotations.shape) 1024 if ( 1025 self.mask is not None 1026 and video_id in self.mask["masked"] 1027 and ind in self.mask["masked"][video_id] 1028 ): 1029 for start, end in self.mask["masked"][video_id][ind]: 1030 masked[..., int(start) : int(end)] = 1 1031 for i, start in enumerate(sp): 1032 self.original_coordinates.append((v_id, i)) 1033 if self.exclusive: 1034 ann = annotations[start : start + self.len_segment] 1035 m = masked[start : start + self.len_segment] 1036 else: 1037 ann = annotations[:, start : start + self.len_segment] 1038 m = masked[:, start : start + self.len_segment] 1039 labels.append(ann) 1040 masked_all.append(m) 1041 self.original_coordinates = np.array(self.original_coordinates) 1042 labels = torch.tensor(np.array(labels)) 1043 masked_all = torch.tensor(np.array(masked_all)).int().bool() 1044 if self.filter_background and not self.exclusive: 1045 for i, label in enumerate(labels): 1046 label[:, torch.sum((label == 1) | (label == 3), 0) == 0] = -100 1047 label[label == 3] = 0 1048 labels[(labels != -100) & masked_all] = -200 1049 return labels 1050 1051 @classmethod 1052 def _get_file_paths(cls, annotation_path: Union[str, Set]) -> List: 1053 """ 1054 Get a list of relevant files 1055 """ 1056 1057 file_paths = [] 1058 if annotation_path is not None: 1059 if isinstance(annotation_path, str): 1060 annotation_path = [annotation_path] 1061 for folder in annotation_path: 1062 file_paths += [os.path.join(folder, x) for x in os.listdir(folder)] 1063 return file_paths 1064 1065 def _get_max_frame(self, video_id: str, clip_id: str): 1066 """ 1067 Get the end frame of a clip in a video 1068 """ 1069 1070 if clip_id in self.max_frames[video_id]: 1071 return self.max_frames[video_id][clip_id] 1072 else: 1073 return min( 1074 [self.max_frames[video_id][ind_k] for ind_k in clip_id.split("+")] 1075 ) 1076 1077 def _get_min_frame(self, video_id, clip_id): 1078 """ 1079 Get the start frame of a clip in a video 1080 """ 1081 1082 if clip_id in self.min_frames[video_id]: 1083 return self.min_frames[video_id][clip_id] 1084 else: 1085 return max( 1086 [self.min_frames[video_id][ind_k] for ind_k in clip_id.split("+")] 1087 ) 1088 1089 @abstractmethod 1090 def _load_data(self) -> torch.Tensor: 1091 """ 1092 Load behavior annotation and generate annotation prompts 1093 """ 1094 1095 1096class FileAnnotationStore(ActionSegmentationStore): # + 1097 """ 1098 A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to one video 1099 """ 1100 1101 def _generate_max_min_frames(self, times: Dict, video_id: str) -> None: 1102 """ 1103 Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore` 1104 """ 1105 1106 if video_id in self.max_frames: 1107 return 1108 for ind, cat_dict in times.items(): 1109 maxes = [] 1110 mins = [] 1111 for cat, cat_list in cat_dict.items(): 1112 if len(cat_list) > 0: 1113 maxes.append(max([x[1] for x in cat_list])) 1114 mins.append(min([x[0] for x in cat_list])) 1115 self.max_frames[video_id][ind] = max(maxes) 1116 self.min_frames[video_id][ind] = min(mins) 1117 1118 def _load_data(self) -> torch.Tensor: 1119 """ 1120 Load behavior annotation and generate annotation prompts 1121 """ 1122 1123 if self.video_order is None: 1124 return None 1125 1126 files = [] 1127 for x in self.video_order: 1128 ok = False 1129 for folder in self.annotation_folder: 1130 for s in self.ann_suffix: 1131 file = os.path.join(folder, x + s) 1132 if os.path.exists(file): 1133 files.append(file) 1134 ok = True 1135 break 1136 if not ok: 1137 files.append(None) 1138 self.not_found = set() 1139 annotations_dict = {} 1140 print("Computing annotation arrays...") 1141 for name, filename in tqdm(list(zip(self.video_order, files))): 1142 if filename is not None: 1143 times = self._open_annotations(filename) 1144 else: 1145 times = None 1146 if times is not None: 1147 self._generate_max_min_frames(times, name) 1148 annotations_dict.update(self._generate_annotation(times, name)) 1149 del times 1150 annotation = self._make_trimmed_annotations(annotations_dict) 1151 del annotations_dict 1152 if self.filter_annotated: 1153 if self.exclusive: 1154 s = torch.sum((annotation != -100), dim=1) 1155 else: 1156 s = torch.sum( 1157 torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1 1158 ) 1159 self.filtered += torch.where(s == 0)[0].tolist() 1160 annotation[annotation == -200] = -100 1161 return annotation 1162 1163 @abstractmethod 1164 def _open_annotations(self, filename: str) -> Dict: 1165 """ 1166 Load the annotation from filename 1167 1168 Parameters 1169 ---------- 1170 filename : str 1171 path to an annotation file 1172 1173 Returns 1174 ------- 1175 times : dict 1176 a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are 1177 lists of (start, end, ambiguity status) lists 1178 """ 1179 1180 1181class SequenceAnnotationStore(ActionSegmentationStore): # + 1182 """ 1183 A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to multiple videos 1184 """ 1185 1186 def _generate_max_min_frames(self, times: Dict) -> None: 1187 """ 1188 Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore` 1189 """ 1190 1191 for video_id in times: 1192 if video_id in self.max_frames: 1193 continue 1194 self.max_frames[video_id] = {} 1195 for ind, cat_dict in times[video_id].items(): 1196 maxes = [] 1197 mins = [] 1198 for cat, cat_list in cat_dict.items(): 1199 maxes.append(max([x[1] for x in cat_list])) 1200 mins.append(min([x[0] for x in cat_list])) 1201 self.max_frames[video_id][ind] = max(maxes) 1202 self.min_frames[video_id][ind] = min(mins) 1203 1204 @classmethod 1205 def get_file_ids( 1206 cls, 1207 filenames: List = None, 1208 annotation_path: str = None, 1209 *args, 1210 **kwargs, 1211 ) -> List: 1212 """ 1213 Process data parameters and return a list of ids of the videos that should 1214 be processed by the __init__ function 1215 1216 Parameters 1217 ---------- 1218 filenames : list, optional 1219 a list of annotation file paths 1220 annotation_path : str, optional 1221 path to the annotation folder 1222 1223 Returns 1224 ------- 1225 video_ids : list 1226 a list of video file ids 1227 """ 1228 1229 file_paths = [] 1230 if annotation_path is not None: 1231 if isinstance(annotation_path, str): 1232 annotation_path = [annotation_path] 1233 file_paths = [] 1234 for folder in annotation_path: 1235 file_paths += [os.path.join(folder, x) for x in os.listdir(folder)] 1236 ids = set() 1237 for f in file_paths: 1238 if os.path.basename(f) in filenames: 1239 ids.add(os.path.basename(f)) 1240 ids = sorted(ids) 1241 return ids 1242 1243 def _load_data(self) -> torch.Tensor: 1244 """ 1245 Load behavior annotation and generate annotation prompts 1246 """ 1247 1248 if self.video_order is None: 1249 return None 1250 1251 files = [] 1252 for f in self.file_paths: 1253 if os.path.basename(f) in self.video_order: 1254 files.append(f) 1255 self.not_found = set() 1256 annotations_dict = {} 1257 for name, filename in tqdm(zip(self.video_order, files)): 1258 if filename is not None: 1259 times = self._open_sequences(filename) 1260 else: 1261 times = None 1262 if times is not None: 1263 self._generate_max_min_frames(times) 1264 none_ids = [] 1265 for video_id, sequence_dict in times.items(): 1266 if sequence_dict is None: 1267 none_ids.append(sequence_dict) 1268 continue 1269 annotations_dict.update( 1270 self._generate_annotation(sequence_dict, video_id) 1271 ) 1272 for video_id in none_ids: 1273 annotations_dict.update(self._generate_annotation(None, video_id)) 1274 del times 1275 annotation = self._make_trimmed_annotations(annotations_dict) 1276 del annotations_dict 1277 if self.filter_annotated: 1278 if self.exclusive: 1279 s = torch.sum((annotation != -100), dim=1) 1280 else: 1281 s = torch.sum( 1282 torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1 1283 ) 1284 self.filtered += torch.where(s == 0)[0].tolist() 1285 annotation[annotation == -200] = -100 1286 return annotation 1287 1288 @abstractmethod 1289 def _open_sequences(self, filename: str) -> Dict: 1290 """ 1291 Load the annotation from filename 1292 1293 Parameters 1294 ---------- 1295 filename : str 1296 path to an annotation file 1297 1298 Returns 1299 ------- 1300 times : dict 1301 a nested dictionary where first-level keys are video ids, second-level keys are clip ids, 1302 third-level keys are categories and values are 1303 lists of (start, end, ambiguity status) lists 1304 """ 1305 1306 1307class DLCAnnotationStore(FileAnnotationStore): # + 1308 """ 1309 DLC type annotation data 1310 1311 The files are either the DLC2Action GUI output or a pickled dictionary of the following structure: 1312 - nested dictionary, 1313 - first-level keys are individual IDs, 1314 - second-level keys are labels, 1315 - values are lists of intervals, 1316 - the lists of intervals is formatted as `[start_frame, end_frame, ambiguity]`, 1317 - ambiguity is 1 if the action is ambiguous (!!at the moment DLC2Action will IGNORE those intervals!!) or 0 if it isn't. 1318 1319 A minimum working example of such a dictionary is: 1320 ``` 1321 { 1322 "ind0": {}, 1323 "ind1": { 1324 "running": [60, 70, 0]], 1325 "eating": [] 1326 } 1327 } 1328 ``` 1329 1330 Here there are two animals: `"ind0"` and `"ind1"`, and two actions: running and eating. 1331 The only annotated action is eating for `"ind1"` between frames 60 and 70. 1332 1333 If you generate those files manually, run this code for a sanity check: 1334 ``` 1335 import pickle 1336 1337 with open("/path/to/annotation.pickle", "rb") as f: 1338 data = pickle.load(f) 1339 1340 for ind, ind_dict in data.items(): 1341 print(f'individual {ind}:') 1342 for label, intervals in ind_dict.items(): 1343 for start, end, ambiguity in intervals: 1344 if ambiguity == 0: 1345 print(f' from {start} to {end} frame: {label}') 1346 ``` 1347 1348 Assumes the following file structure: 1349 ``` 1350 annotation_path 1351 ├── video1_annotation.pickle 1352 └── video2_labels.pickle 1353 ``` 1354 Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`. 1355 """ 1356 1357 def _open_annotations(self, filename: str) -> Dict: 1358 """ 1359 Load the annotation from `filename` 1360 """ 1361 1362 try: 1363 with open(filename, "rb") as f: 1364 data = pickle.load(f) 1365 if isinstance(data, dict): 1366 annotation = data 1367 for ind in annotation: 1368 for cat, cat_list in annotation[ind].items(): 1369 annotation[ind][cat] = [ 1370 [start, end, 0] for start, end in cat_list 1371 ] 1372 else: 1373 _, loaded_labels, animals, loaded_times = data 1374 annotation = {} 1375 for ind, ind_list in zip(animals, loaded_times): 1376 annotation[ind] = {} 1377 for cat, cat_list in zip(loaded_labels, ind_list): 1378 annotation[ind][cat] = cat_list 1379 return annotation 1380 except: 1381 print(f"{filename} is invalid or does not exist") 1382 return None 1383 1384 1385class BorisAnnotationStore(FileAnnotationStore): # + 1386 """ 1387 BORIS type annotation data 1388 1389 Assumes the following file structure: 1390 ``` 1391 annotation_path 1392 ├── video1_annotation.pickle 1393 └── video2_labels.pickle 1394 ``` 1395 Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`. 1396 """ 1397 1398 def __init__( 1399 self, 1400 video_order: List = None, 1401 min_frames: Dict = None, 1402 max_frames: Dict = None, 1403 visibility: Dict = None, 1404 exclusive: bool = True, 1405 len_segment: int = 128, 1406 overlap: int = 0, 1407 behaviors: Set = None, 1408 ignored_classes: Set = None, 1409 annotation_suffix: Union[Set, str] = None, 1410 annotation_path: Union[Set, str] = None, 1411 behavior_file: str = None, 1412 correction: Dict = None, 1413 frame_limit: int = 0, 1414 filter_annotated: bool = False, 1415 filter_background: bool = False, 1416 error_class: str = None, 1417 min_frames_action: int = None, 1418 key_objects: Tuple = None, 1419 visibility_min_score: float = 0.2, 1420 visibility_min_frac: float = 0.7, 1421 mask: Dict = None, 1422 use_hard_negatives: bool = False, 1423 default_agent_name: str = "ind0", 1424 interactive: bool = False, 1425 ignored_clips: Set = None, 1426 *args, 1427 **kwargs, 1428 ) -> None: 1429 """ 1430 Parameters 1431 ---------- 1432 video_order : list, optional 1433 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1434 min_frames : dict, optional 1435 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1436 clip start frames (not passed if creating from key objects) 1437 max_frames : dict, optional 1438 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1439 clip end frames (not passed if creating from key objects) 1440 visibility : dict, optional 1441 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1442 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 1443 exclusive : bool, default True 1444 if True, the annotation is single-label; if False, multi-label 1445 len_segment : int, default 128 1446 the length of the segments in which the data should be cut (in frames) 1447 overlap : int, default 0 1448 the length of the overlap between neighboring segments (in frames) 1449 behaviors : set, optional 1450 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 1451 loaded from a file) 1452 ignored_classes : set, optional 1453 the list of behaviors from the behaviors list or file to not annotate 1454 annotation_suffix : str | set, optional 1455 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 1456 (not passed if creating from key objects or if irrelevant for the dataset) 1457 annotation_path : str | set, optional 1458 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1459 from key objects) 1460 behavior_file : str, optional 1461 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 1462 correction : dict, optional 1463 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 1464 can be used to correct for variations in naming or to merge several labels in one 1465 frame_limit : int, default 0 1466 the smallest possible length of a clip (shorter clips are discarded) 1467 filter_annotated : bool, default False 1468 if True, the samples that do not have any labels will be filtered 1469 filter_background : bool, default False 1470 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 1471 error_class : str, optional 1472 the name of the error class (the annotations that intersect with this label will be discarded) 1473 min_frames_action : int, default 0 1474 the minimum length of an action (shorter actions are not annotated) 1475 key_objects : tuple, optional 1476 the key objects to load the AnnotationStore from 1477 visibility_min_score : float, default 5 1478 the minimum visibility score for visibility filtering 1479 visibility_min_frac : float, default 0.7 1480 the minimum fraction of visible frames for visibility filtering 1481 mask : dict, optional 1482 a masked value dictionary (for active learning simulation experiments) 1483 use_hard_negatives : bool, default False 1484 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 1485 interactive : bool, default False 1486 if `True`, annotation is assigned to pairs of individuals 1487 ignored_clips : set, optional 1488 a set of clip ids to ignore 1489 """ 1490 1491 self.default_agent_name = default_agent_name 1492 super().__init__( 1493 video_order=video_order, 1494 min_frames=min_frames, 1495 max_frames=max_frames, 1496 visibility=visibility, 1497 exclusive=exclusive, 1498 len_segment=len_segment, 1499 overlap=overlap, 1500 behaviors=behaviors, 1501 ignored_classes=ignored_classes, 1502 annotation_suffix=annotation_suffix, 1503 annotation_path=annotation_path, 1504 behavior_file=behavior_file, 1505 correction=correction, 1506 frame_limit=frame_limit, 1507 filter_annotated=filter_annotated, 1508 filter_background=filter_background, 1509 error_class=error_class, 1510 min_frames_action=min_frames_action, 1511 key_objects=key_objects, 1512 visibility_min_score=visibility_min_score, 1513 visibility_min_frac=visibility_min_frac, 1514 mask=mask, 1515 use_hard_negatives=use_hard_negatives, 1516 interactive=interactive, 1517 ignored_clips=ignored_clips, 1518 ) 1519 1520 def _open_annotations(self, filename: str) -> Dict: 1521 """ 1522 Load the annotation from filename 1523 """ 1524 1525 try: 1526 df = pd.read_csv(filename, header=15) 1527 fps = df.iloc[0]["FPS"] 1528 df["Subject"] = df["Subject"].fillna(self.default_agent_name) 1529 loaded_labels = list(df["Behavior"].unique()) 1530 animals = list(df["Subject"].unique()) 1531 loaded_times = {} 1532 for ind in animals: 1533 loaded_times[ind] = {} 1534 agent_df = df[df["Subject"] == ind] 1535 for cat in loaded_labels: 1536 filtered_df = agent_df[agent_df["Behavior"] == cat] 1537 starts = ( 1538 filtered_df["Time"][filtered_df["Status"] == "START"] * fps 1539 ).astype(int) 1540 ends = ( 1541 filtered_df["Time"][filtered_df["Status"] == "STOP"] * fps 1542 ).astype(int) 1543 loaded_times[ind][cat] = [ 1544 [start, end, 0] for start, end in zip(starts, ends) 1545 ] 1546 return loaded_times 1547 except: 1548 print(f"{filename} is invalid or does not exist") 1549 return None 1550 1551 1552class PKUMMDAnnotationStore(FileAnnotationStore): # + 1553 """ 1554 PKU-MMD annotation data 1555 1556 Assumes the following file structure: 1557 ``` 1558 annotation_path 1559 ├── 0364-L.txt 1560 ... 1561 └── 0144-M.txt 1562 ``` 1563 """ 1564 1565 def __init__( 1566 self, 1567 video_order: List = None, 1568 min_frames: Dict = None, 1569 max_frames: Dict = None, 1570 visibility: Dict = None, 1571 exclusive: bool = True, 1572 len_segment: int = 128, 1573 overlap: int = 0, 1574 ignored_classes: Set = None, 1575 annotation_path: Union[Set, str] = None, 1576 behavior_file: str = None, 1577 correction: Dict = None, 1578 frame_limit: int = 0, 1579 filter_annotated: bool = False, 1580 filter_background: bool = False, 1581 error_class: str = None, 1582 min_frames_action: int = None, 1583 key_objects: Tuple = None, 1584 visibility_min_score: float = 0, 1585 visibility_min_frac: float = 0, 1586 mask: Dict = None, 1587 use_hard_negatives: bool = False, 1588 interactive: bool = False, 1589 ignored_clips: Set = None, 1590 *args, 1591 **kwargs, 1592 ) -> None: 1593 """ 1594 Parameters 1595 ---------- 1596 video_order : list, optional 1597 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1598 min_frames : dict, optional 1599 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1600 clip start frames (not passed if creating from key objects) 1601 max_frames : dict, optional 1602 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1603 clip end frames (not passed if creating from key objects) 1604 visibility : dict, optional 1605 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1606 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 1607 exclusive : bool, default True 1608 if True, the annotation is single-label; if False, multi-label 1609 len_segment : int, default 128 1610 the length of the segments in which the data should be cut (in frames) 1611 overlap : int, default 0 1612 the length of the overlap between neighboring segments (in frames) 1613 ignored_classes : set, optional 1614 the list of behaviors from the behaviors list or file to not annotate 1615 annotation_path : str | set, optional 1616 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1617 from key objects) 1618 behavior_file : str, optional 1619 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 1620 correction : dict, optional 1621 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 1622 can be used to correct for variations in naming or to merge several labels in one 1623 frame_limit : int, default 0 1624 the smallest possible length of a clip (shorter clips are discarded) 1625 filter_annotated : bool, default False 1626 if True, the samples that do not have any labels will be filtered 1627 filter_background : bool, default False 1628 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 1629 error_class : str, optional 1630 the name of the error class (the annotations that intersect with this label will be discarded) 1631 min_frames_action : int, default 0 1632 the minimum length of an action (shorter actions are not annotated) 1633 key_objects : tuple, optional 1634 the key objects to load the AnnotationStore from 1635 visibility_min_score : float, default 5 1636 the minimum visibility score for visibility filtering 1637 visibility_min_frac : float, default 0.7 1638 the minimum fraction of visible frames for visibility filtering 1639 mask : dict, optional 1640 a masked value dictionary (for active learning simulation experiments) 1641 use_hard_negatives : bool, default False 1642 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 1643 interactive : bool, default False 1644 if `True`, annotation is assigned to pairs of individuals 1645 ignored_clips : set, optional 1646 a set of clip ids to ignore 1647 """ 1648 1649 super().__init__( 1650 video_order=video_order, 1651 min_frames=min_frames, 1652 max_frames=max_frames, 1653 visibility=visibility, 1654 exclusive=exclusive, 1655 len_segment=len_segment, 1656 overlap=overlap, 1657 behaviors=None, 1658 ignored_classes=ignored_classes, 1659 annotation_suffix={".txt"}, 1660 annotation_path=annotation_path, 1661 behavior_file=behavior_file, 1662 correction=correction, 1663 frame_limit=frame_limit, 1664 filter_annotated=filter_annotated, 1665 filter_background=filter_background, 1666 error_class=error_class, 1667 min_frames_action=min_frames_action, 1668 key_objects=key_objects, 1669 visibility_min_score=visibility_min_score, 1670 visibility_min_frac=visibility_min_frac, 1671 mask=mask, 1672 use_hard_negatives=use_hard_negatives, 1673 interactive=interactive, 1674 ignored_clips=ignored_clips, 1675 *args, 1676 **kwargs, 1677 ) 1678 1679 @classmethod 1680 def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List: 1681 """ 1682 Process data parameters and return a list of ids of the videos that should 1683 be processed by the __init__ function 1684 1685 Parameters 1686 ---------- 1687 annotation_path : str | set 1688 the path or the set of paths to the folder where the annotation files are stored 1689 1690 Returns 1691 ------- 1692 video_ids : list 1693 a list of video file ids 1694 """ 1695 1696 if isinstance(annotation_path, str): 1697 annotation_path = [annotation_path] 1698 files = [] 1699 for folder in annotation_path: 1700 files += [ 1701 os.path.basename(x)[:-4] for x in os.listdir(folder) if x[-4:] == ".txt" 1702 ] 1703 files = sorted(files, key=lambda x: os.path.basename(x)) 1704 return files 1705 1706 def _open_annotations(self, filename: str) -> Dict: 1707 """ 1708 Load the annotation from filename 1709 """ 1710 1711 if self.interactive: 1712 agent_name = "0+1" 1713 else: 1714 agent_name = "0" 1715 times = {agent_name: defaultdict(lambda: [])} 1716 with open(filename) as f: 1717 for line in f.readlines(): 1718 label, start, end, *_ = map(int, line.split(",")) 1719 times[agent_name][self.all_behaviors[int(label) - 1]].append( 1720 [start, end, 0] 1721 ) 1722 return times 1723 1724 def _set_behaviors( 1725 self, behaviors: List, ignored_classes: List, behavior_file: str 1726 ): 1727 """ 1728 Get a list of behaviors that should be annotated from behavior parameters 1729 """ 1730 1731 if behavior_file is not None: 1732 behaviors = list(pd.read_excel(behavior_file)["Action"]) 1733 self.all_behaviors = copy(behaviors) 1734 for b in ignored_classes: 1735 if b in behaviors: 1736 behaviors.remove(b) 1737 self.behaviors = behaviors 1738 1739 1740class CalMS21AnnotationStore(SequenceAnnotationStore): # + 1741 """ 1742 CalMS21 annotation data 1743 1744 Use the `'random:test_from_name:{name}'` and `'val-from-name:{val_name}:test-from-name:{test_name}'` 1745 partitioning methods with `'train'`, `'test'` and `'unlabeled'` names to separate into train, test and validation 1746 subsets according to the original files. For example, with `'val-from-name:test:test-from-name:unlabeled'` 1747 the data from the test file will go into validation and the unlabeled files will be the test. 1748 1749 Assumes the following file structure: 1750 ``` 1751 annotation_path 1752 ├── calms21_task_train.npy 1753 ├── calms21_task_test.npy 1754 ├── calms21_unlabeled_videos_part1.npy 1755 ├── calms21_unlabeled_videos_part2.npy 1756 └── calms21_unlabeled_videos_part3.npy 1757 ``` 1758 """ 1759 1760 def __init__( 1761 self, 1762 task_n: int = 1, 1763 include_task1: bool = True, 1764 video_order: List = None, 1765 min_frames: Dict = None, 1766 max_frames: Dict = None, 1767 len_segment: int = 128, 1768 overlap: int = 0, 1769 ignored_classes: Set = None, 1770 annotation_path: Union[Set, str] = None, 1771 key_objects: Tuple = None, 1772 treba_files: bool = False, 1773 *args, 1774 **kwargs, 1775 ) -> None: 1776 """ 1777 1778 Parameters 1779 ---------- 1780 task_n : [1, 2] 1781 the number of the task 1782 include_task1 : bool, default True 1783 include task 1 data to training set 1784 video_order : list, optional 1785 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1786 min_frames : dict, optional 1787 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1788 clip start frames (not passed if creating from key objects) 1789 max_frames : dict, optional 1790 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1791 clip end frames (not passed if creating from key objects) 1792 len_segment : int, default 128 1793 the length of the segments in which the data should be cut (in frames) 1794 overlap : int, default 0 1795 the length of the overlap between neighboring segments (in frames) 1796 ignored_classes : set, optional 1797 the list of behaviors from the behaviors list or file to not annotate 1798 annotation_path : str | set, optional 1799 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1800 from key objects) 1801 key_objects : tuple, optional 1802 the key objects to load the AnnotationStore from 1803 treba_files : bool, default False 1804 if `True`, TREBA feature files will be loaded 1805 """ 1806 1807 self.task_n = int(task_n) 1808 self.include_task1 = include_task1 1809 if self.task_n == 1: 1810 self.include_task1 = False 1811 self.treba_files = treba_files 1812 if "exclusive" in kwargs: 1813 exclusive = kwargs["exclusive"] 1814 else: 1815 exclusive = True 1816 if "behaviors" in kwargs and kwargs["behaviors"] is not None: 1817 behaviors = kwargs["behaviors"] 1818 else: 1819 behaviors = ["attack", "investigation", "mount", "other"] 1820 if task_n == 3: 1821 exclusive = False 1822 behaviors += [ 1823 "approach", 1824 "disengaged", 1825 "groom", 1826 "intromission", 1827 "mount_attempt", 1828 "sniff_face", 1829 "whiterearing", 1830 ] 1831 super().__init__( 1832 video_order=video_order, 1833 min_frames=min_frames, 1834 max_frames=max_frames, 1835 exclusive=exclusive, 1836 len_segment=len_segment, 1837 overlap=overlap, 1838 behaviors=behaviors, 1839 ignored_classes=ignored_classes, 1840 annotation_path=annotation_path, 1841 key_objects=key_objects, 1842 filter_annotated=False, 1843 interactive=True, 1844 ) 1845 1846 @classmethod 1847 def get_file_ids( 1848 cls, 1849 task_n: int = 1, 1850 include_task1: bool = False, 1851 treba_files: bool = False, 1852 annotation_path: Union[str, Set] = None, 1853 file_paths=None, 1854 *args, 1855 **kwargs, 1856 ) -> Iterable: 1857 """ 1858 Process data parameters and return a list of ids of the videos that should 1859 be processed by the __init__ function 1860 1861 Parameters 1862 ---------- 1863 task_n : {1, 2, 3} 1864 the index of the CalMS21 challenge task 1865 include_task1 : bool, default False 1866 if `True`, the training file of the task 1 will be loaded 1867 treba_files : bool, default False 1868 if `True`, the TREBA feature files will be loaded 1869 filenames : set, optional 1870 a set of string filenames to search for (only basenames, not the whole paths) 1871 annotation_path : str | set, optional 1872 the path to the folder where the pose and feature files are stored or a set of such paths 1873 (not passed if creating from key objects or from `file_paths`) 1874 file_paths : set, optional 1875 a set of string paths to the pose and feature files 1876 (not passed if creating from key objects or from `data_path`) 1877 1878 Returns 1879 ------- 1880 video_ids : list 1881 a list of video file ids 1882 """ 1883 1884 task_n = int(task_n) 1885 if task_n == 1: 1886 include_task1 = False 1887 files = [] 1888 if treba_files: 1889 postfix = "_features" 1890 else: 1891 postfix = "" 1892 files.append(f"calms21_task{task_n}_train{postfix}.npy") 1893 files.append(f"calms21_task{task_n}_test{postfix}.npy") 1894 if include_task1: 1895 files.append(f"calms21_task1_train{postfix}.npy") 1896 filenames = set(files) 1897 return SequenceAnnotationStore.get_file_ids( 1898 filenames, annotation_path=annotation_path 1899 ) 1900 1901 def _open_sequences(self, filename: str) -> Dict: 1902 """ 1903 Load the annotation from filename 1904 1905 Parameters 1906 ---------- 1907 filename : str 1908 path to an annotation file 1909 1910 Returns 1911 ------- 1912 times : dict 1913 a nested dictionary where first-level keys are video ids, second-level keys are clip ids, 1914 third-level keys are categories and values are 1915 lists of (start, end, ambiguity status) lists 1916 """ 1917 1918 data_dict = np.load(filename, allow_pickle=True).item() 1919 data = {} 1920 result = {} 1921 keys = list(data_dict.keys()) 1922 if "test" in os.path.basename(filename): 1923 mode = "test" 1924 elif "unlabeled" in os.path.basename(filename): 1925 mode = "unlabeled" 1926 else: 1927 mode = "train" 1928 if "approach" in keys: 1929 for behavior in keys: 1930 for key in data_dict[behavior].keys(): 1931 ann = data_dict[behavior][key]["annotations"] 1932 result[f'{mode}--{key.split("/")[-1]}'] = { 1933 "mouse1+mouse2": defaultdict(lambda: []) 1934 } 1935 starts = np.where( 1936 np.diff(np.concatenate([np.array([0]), ann, np.array([0])])) 1937 == 1 1938 )[0] 1939 ends = np.where( 1940 np.diff(np.concatenate([np.array([0]), ann, np.array([0])])) 1941 == -1 1942 )[0] 1943 for start, end in zip(starts, ends): 1944 result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][ 1945 behavior 1946 ].append([start, end, 0]) 1947 for b in self.behaviors: 1948 result[f'{mode}--{key.split("/")[-1]}---mouse1+mouse2'][ 1949 "mouse1+mouse2" 1950 ][f"unknown {b}"].append([0, len(ann), 0]) 1951 for key in keys: 1952 data.update(data_dict[key]) 1953 data_dict.pop(key) 1954 if "approach" not in keys and self.task_n == 3: 1955 for key in data.keys(): 1956 result[f'{mode}--{key.split("/")[-1]}'] = {"mouse1+mouse2": {}} 1957 ann = data[key]["annotations"] 1958 for i in range(4): 1959 starts = np.where( 1960 np.diff( 1961 np.concatenate( 1962 [np.array([0]), (ann == i).astype(int), np.array([0])] 1963 ) 1964 ) 1965 == 1 1966 )[0] 1967 ends = np.where( 1968 np.diff( 1969 np.concatenate( 1970 [np.array([0]), (ann == i).astype(int), np.array([0])] 1971 ) 1972 ) 1973 == -1 1974 )[0] 1975 result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][ 1976 self.behaviors_dict()[i] 1977 ] = [[start, end, 0] for start, end in zip(starts, ends)] 1978 if self.task_n != 3: 1979 for seq_name, seq_dict in data.items(): 1980 if "annotations" not in seq_dict: 1981 return None 1982 behaviors = np.unique(seq_dict["annotations"]) 1983 ann = seq_dict["annotations"] 1984 key = f'{mode}--{seq_name.split("/")[-1]}' 1985 result[key] = {"mouse1+mouse2": {}} 1986 for i in behaviors: 1987 starts = np.where( 1988 np.diff( 1989 np.concatenate( 1990 [np.array([0]), (ann == i).astype(int), np.array([0])] 1991 ) 1992 ) 1993 == 1 1994 )[0] 1995 ends = np.where( 1996 np.diff( 1997 np.concatenate( 1998 [np.array([0]), (ann == i).astype(int), np.array([0])] 1999 ) 2000 ) 2001 == -1 2002 )[0] 2003 result[key]["mouse1+mouse2"][self.behaviors_dict()[i]] = [ 2004 [start, end, 0] for start, end in zip(starts, ends) 2005 ] 2006 return result 2007 2008 2009class CSVAnnotationStore(FileAnnotationStore): # + 2010 """ 2011 CSV type annotation data 2012 2013 Assumes that files are saved as .csv tables with at least the following columns: 2014 - from / start : start of action, 2015 - to / end : end of action, 2016 - class / behavior / behaviour / label / type : action label. 2017 2018 If the times are set in seconds instead of frames, don't forget to set the `fps` parameter to your frame rate. 2019 2020 Assumes the following file structure: 2021 ``` 2022 annotation_path 2023 ├── video1_annotation.csv 2024 └── video2_labels.csv 2025 ``` 2026 Here `annotation_suffix` is `{'_annotation.csv', '_labels.csv'}`. 2027 """ 2028 2029 def __init__( 2030 self, 2031 video_order: List = None, 2032 min_frames: Dict = None, 2033 max_frames: Dict = None, 2034 visibility: Dict = None, 2035 exclusive: bool = True, 2036 len_segment: int = 128, 2037 overlap: int = 0, 2038 behaviors: Set = None, 2039 ignored_classes: Set = None, 2040 annotation_suffix: Union[Set, str] = None, 2041 annotation_path: Union[Set, str] = None, 2042 behavior_file: str = None, 2043 correction: Dict = None, 2044 frame_limit: int = 0, 2045 filter_annotated: bool = False, 2046 filter_background: bool = False, 2047 error_class: str = None, 2048 min_frames_action: int = None, 2049 key_objects: Tuple = None, 2050 visibility_min_score: float = 0.2, 2051 visibility_min_frac: float = 0.7, 2052 mask: Dict = None, 2053 default_agent_name: str = "ind0", 2054 separator: str = ",", 2055 fps: int = 30, 2056 *args, 2057 **kwargs, 2058 ) -> None: 2059 """ 2060 Parameters 2061 ---------- 2062 video_order : list, optional 2063 a list of video ids that should be processed in the same order (not passed if creating from key objects) 2064 min_frames : dict, optional 2065 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2066 clip start frames (not passed if creating from key objects) 2067 max_frames : dict, optional 2068 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2069 clip end frames (not passed if creating from key objects) 2070 visibility : dict, optional 2071 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2072 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 2073 exclusive : bool, default True 2074 if True, the annotation is single-label; if False, multi-label 2075 len_segment : int, default 128 2076 the length of the segments in which the data should be cut (in frames) 2077 overlap : int, default 0 2078 the length of the overlap between neighboring segments (in frames) 2079 behaviors : set, optional 2080 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 2081 loaded from a file) 2082 ignored_classes : set, optional 2083 the list of behaviors from the behaviors list or file to not annotate 2084 annotation_suffix : str | set, optional 2085 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 2086 (not passed if creating from key objects or if irrelevant for the dataset) 2087 annotation_path : str | set, optional 2088 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 2089 from key objects) 2090 behavior_file : str, optional 2091 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 2092 correction : dict, optional 2093 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 2094 can be used to correct for variations in naming or to merge several labels in one 2095 frame_limit : int, default 0 2096 the smallest possible length of a clip (shorter clips are discarded) 2097 filter_annotated : bool, default False 2098 if True, the samples that do not have any labels will be filtered 2099 filter_background : bool, default False 2100 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 2101 error_class : str, optional 2102 the name of the error class (the annotations that intersect with this label will be discarded) 2103 min_frames_action : int, default 0 2104 the minimum length of an action (shorter actions are not annotated) 2105 key_objects : tuple, optional 2106 the key objects to load the AnnotationStore from 2107 visibility_min_score : float, default 5 2108 the minimum visibility score for visibility filtering 2109 visibility_min_frac : float, default 0.7 2110 the minimum fraction of visible frames for visibility filtering 2111 mask : dict, optional 2112 a masked value dictionary (for active learning simulation experiments) 2113 default_agent_name : str, default "ind0" 2114 the clip id to use when there is no given 2115 separator : str, default "," 2116 the separator in the csv files 2117 fps : int, default 30 2118 frames per second in the videos 2119 """ 2120 2121 self.default_agent_name = default_agent_name 2122 self.separator = separator 2123 self.fps = fps 2124 super().__init__( 2125 video_order=video_order, 2126 min_frames=min_frames, 2127 max_frames=max_frames, 2128 visibility=visibility, 2129 exclusive=exclusive, 2130 len_segment=len_segment, 2131 overlap=overlap, 2132 behaviors=behaviors, 2133 ignored_classes=ignored_classes, 2134 ignored_clips=None, 2135 annotation_suffix=annotation_suffix, 2136 annotation_path=annotation_path, 2137 behavior_file=behavior_file, 2138 correction=correction, 2139 frame_limit=frame_limit, 2140 filter_annotated=filter_annotated, 2141 filter_background=filter_background, 2142 error_class=error_class, 2143 min_frames_action=min_frames_action, 2144 key_objects=key_objects, 2145 visibility_min_score=visibility_min_score, 2146 visibility_min_frac=visibility_min_frac, 2147 mask=mask, 2148 ) 2149 2150 def _open_annotations(self, filename: str) -> Dict: 2151 """ 2152 Load the annotation from `filename` 2153 """ 2154 2155 # try: 2156 data = pd.read_csv(filename, sep=self.separator) 2157 data.columns = list(map(lambda x: x.lower(), data.columns)) 2158 starts, ends, actions = None, None, None 2159 start_names = ["from", "start"] 2160 for x in start_names: 2161 if x in data.columns: 2162 starts = data[x] 2163 end_names = ["to", "end"] 2164 for x in end_names: 2165 if x in data.columns: 2166 ends = data[x] 2167 class_names = ["class", "behavior", "behaviour", "type", "label"] 2168 for x in class_names: 2169 if x in data.columns: 2170 actions = data[x] 2171 if starts is None: 2172 raise ValueError("The file must have a column titled 'from' or 'start'!") 2173 if ends is None: 2174 raise ValueError("The file must have a column titled 'to' or 'end'!") 2175 if actions is None: 2176 raise ValueError( 2177 "The file must have a column titled 'class', 'behavior', 'behaviour', 'type' or 'label'!" 2178 ) 2179 times = defaultdict(lambda: defaultdict(lambda: [])) 2180 for start, end, action in zip(starts, ends, actions): 2181 if any([np.isnan(x) for x in [start, end]]): 2182 continue 2183 times[self.default_agent_name][action].append( 2184 [int(start * self.fps), int(end * self.fps), 0] 2185 ) 2186 return times 2187 2188 2189class SIMBAAnnotationStore(FileAnnotationStore): # + 2190 """ 2191 SIMBA paper format data 2192 2193 Assumes the following file structure: 2194 ``` 2195 annotation_path 2196 ├── Video1.csv 2197 ... 2198 └── Video9.csv 2199 """ 2200 2201 def __init__( 2202 self, 2203 video_order: List = None, 2204 min_frames: Dict = None, 2205 max_frames: Dict = None, 2206 visibility: Dict = None, 2207 exclusive: bool = True, 2208 len_segment: int = 128, 2209 overlap: int = 0, 2210 behaviors: Set = None, 2211 ignored_classes: Set = None, 2212 ignored_clips: Set = None, 2213 annotation_path: Union[Set, str] = None, 2214 correction: Dict = None, 2215 filter_annotated: bool = False, 2216 filter_background: bool = False, 2217 error_class: str = None, 2218 min_frames_action: int = None, 2219 key_objects: Tuple = None, 2220 visibility_min_score: float = 0.2, 2221 visibility_min_frac: float = 0.7, 2222 mask: Dict = None, 2223 use_hard_negatives: bool = False, 2224 annotation_suffix: str = None, 2225 *args, 2226 **kwargs, 2227 ) -> None: 2228 """ 2229 Parameters 2230 ---------- 2231 video_order : list, optional 2232 a list of video ids that should be processed in the same order (not passed if creating from key objects) 2233 min_frames : dict, optional 2234 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2235 clip start frames (not passed if creating from key objects) 2236 max_frames : dict, optional 2237 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2238 clip end frames (not passed if creating from key objects) 2239 visibility : dict, optional 2240 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2241 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 2242 exclusive : bool, default True 2243 if True, the annotation is single-label; if False, multi-label 2244 len_segment : int, default 128 2245 the length of the segments in which the data should be cut (in frames) 2246 overlap : int, default 0 2247 the length of the overlap between neighboring segments (in frames) 2248 behaviors : set, optional 2249 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 2250 loaded from a file) 2251 ignored_classes : set, optional 2252 the list of behaviors from the behaviors list or file to not annotate 2253 ignored_clips : set, optional 2254 clip ids to ignore 2255 annotation_path : str | set, optional 2256 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 2257 from key objects) 2258 behavior_file : str, optional 2259 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 2260 correction : dict, optional 2261 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 2262 can be used to correct for variations in naming or to merge several labels in one 2263 filter_annotated : bool, default False 2264 if True, the samples that do not have any labels will be filtered 2265 filter_background : bool, default False 2266 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 2267 error_class : str, optional 2268 the name of the error class (the annotations that intersect with this label will be discarded) 2269 min_frames_action : int, default 0 2270 the minimum length of an action (shorter actions are not annotated) 2271 key_objects : tuple, optional 2272 the key objects to load the AnnotationStore from 2273 visibility_min_score : float, default 5 2274 the minimum visibility score for visibility filtering 2275 visibility_min_frac : float, default 0.7 2276 the minimum fraction of visible frames for visibility filtering 2277 mask : dict, optional 2278 a masked value dictionary (for active learning simulation experiments) 2279 use_hard_negatives : bool, default False 2280 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 2281 annotation_suffix : str | set, optional 2282 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 2283 (not passed if creating from key objects or if irrelevant for the dataset) 2284 """ 2285 2286 super().__init__( 2287 video_order=video_order, 2288 min_frames=min_frames, 2289 max_frames=max_frames, 2290 visibility=visibility, 2291 exclusive=exclusive, 2292 len_segment=len_segment, 2293 overlap=overlap, 2294 behaviors=behaviors, 2295 ignored_classes=ignored_classes, 2296 ignored_clips=ignored_clips, 2297 annotation_suffix=annotation_suffix, 2298 annotation_path=annotation_path, 2299 behavior_file=None, 2300 correction=correction, 2301 frame_limit=0, 2302 filter_annotated=filter_annotated, 2303 filter_background=filter_background, 2304 error_class=error_class, 2305 min_frames_action=min_frames_action, 2306 key_objects=key_objects, 2307 visibility_min_score=visibility_min_score, 2308 visibility_min_frac=visibility_min_frac, 2309 mask=mask, 2310 use_hard_negatives=use_hard_negatives, 2311 interactive=True, 2312 ) 2313 2314 def _open_annotations(self, filename: str) -> Dict: 2315 """ 2316 Load the annotation from filename 2317 2318 Parameters 2319 ---------- 2320 filename : str 2321 path to an annotation file 2322 2323 Returns 2324 ------- 2325 times : dict 2326 a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are 2327 lists of (start, end, ambiguity status) lists 2328 """ 2329 2330 data = pd.read_csv(filename) 2331 columns = [x for x in data.columns if x.split("_")[-1] == "x"] 2332 animals = sorted(set([x.split("_")[-2] for x in columns])) 2333 if len(animals) > 2: 2334 raise ValueError( 2335 "SIMBAAnnotationStore is only implemented for files with 1 or 2 animals!" 2336 ) 2337 if len(animals) == 1: 2338 ind = animals[0] 2339 else: 2340 ind = "+".join(animals) 2341 behaviors = [ 2342 "_".join(x.split("_")[:-1]) 2343 for x in data.columns 2344 if x.split("_")[-1] == "prediction" 2345 ] 2346 result = {} 2347 for behavior in behaviors: 2348 ann = data[f"{behavior}_prediction"].values 2349 diff = np.diff( 2350 np.concatenate([np.array([0]), (ann == 1).astype(int), np.array([0])]) 2351 ) 2352 starts = np.where(diff == 1)[0] 2353 ends = np.where(diff == -1)[0] 2354 result[behavior] = [[start, end, 0] for start, end in zip(starts, ends)] 2355 diff = np.diff( 2356 np.concatenate( 2357 [np.array([0]), (np.isnan(ann)).astype(int), np.array([0])] 2358 ) 2359 ) 2360 starts = np.where(diff == 1)[0] 2361 ends = np.where(diff == -1)[0] 2362 result[f"unknown {behavior}"] = [ 2363 [start, end, 0] for start, end in zip(starts, ends) 2364 ] 2365 if self.behaviors is not None: 2366 for behavior in self.behaviors: 2367 if behavior not in behaviors: 2368 result[f"unknown {behavior}"] = [[0, len(data), 0]] 2369 return {ind: result}
28class EmptyAnnotationStore(AnnotationStore): 29 def __init__( 30 self, video_order: List = None, key_objects: Tuple = None, *args, **kwargs 31 ): 32 """ 33 Parameters 34 ---------- 35 video_order : list, optional 36 a list of video ids that should be processed in the same order (not passed if creating from key objects) 37 key_objects : tuple, optional 38 a tuple of key objects 39 """ 40 41 pass 42 43 def __len__(self) -> int: 44 """ 45 Get the number of available samples 46 47 Returns 48 ------- 49 length : int 50 the number of available samples 51 """ 52 53 return 0 54 55 def remove(self, indices: List) -> None: 56 """ 57 Remove the samples corresponding to indices 58 59 Parameters 60 ---------- 61 indices : int 62 a list of integer indices to remove 63 """ 64 65 pass 66 67 def key_objects(self) -> Tuple: 68 """ 69 Return a tuple of the key objects necessary to re-create the Store 70 71 Returns 72 ------- 73 key_objects : tuple 74 a tuple of key objects 75 """ 76 77 return () 78 79 def load_from_key_objects(self, key_objects: Tuple) -> None: 80 """ 81 Load the information from a tuple of key objects 82 83 Parameters 84 ---------- 85 key_objects : tuple 86 a tuple of key objects 87 """ 88 89 pass 90 91 def to_ram(self) -> None: 92 """ 93 Transfer the data samples to RAM if they were previously stored as file paths 94 """ 95 96 pass 97 98 def get_original_coordinates(self) -> np.ndarray: 99 """ 100 Return the original coordinates array 101 102 Returns 103 ------- 104 np.ndarray 105 an array that contains the coordinates of the data samples in original input data (video id, clip id, 106 start frame) 107 """ 108 109 return None 110 111 def create_subsample(self, indices: List, ssl_indices: List = None): 112 """ 113 Create a new store that contains a subsample of the data 114 115 Parameters 116 ---------- 117 indices : list 118 the indices to be included in the subsample 119 ssl_indices : list, optional 120 the indices to be included in the subsample without the annotation data 121 """ 122 123 return self.new() 124 125 @classmethod 126 def get_file_ids(cls, *args, **kwargs) -> List: 127 """ 128 Process data parameters and return a list of ids of the videos that should 129 be processed by the __init__ function 130 131 Returns 132 ------- 133 video_ids : list 134 a list of video file ids 135 """ 136 137 return None 138 139 def __getitem__(self, ind: int) -> torch.Tensor: 140 """ 141 Return the annotation of the sample corresponding to an index 142 143 Parameters 144 ---------- 145 ind : int 146 index of the sample 147 148 Returns 149 ------- 150 sample : torch.Tensor 151 the corresponding annotation tensor 152 """ 153 154 return torch.tensor(float("nan")) 155 156 def get_len(self, return_unlabeled: bool) -> int: 157 """ 158 Get the length of the subsample of labeled/unlabeled data 159 160 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 161 and if return_unlabeled is None the index is already correct 162 163 Parameters 164 ---------- 165 return_unlabeled : bool 166 the identifier for the subsample 167 168 Returns 169 ------- 170 length : int 171 the length of the subsample 172 """ 173 174 return None 175 176 def get_idx(self, index: int, return_unlabeled: bool) -> int: 177 """ 178 Convert from an index in the subsample of labeled/unlabeled data to an index in the full array 179 180 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 181 and if return_unlabeled is None the index is already correct 182 183 Parameters 184 ---------- 185 index : int 186 the index in the subsample 187 return_unlabeled : bool 188 the identifier for the subsample 189 190 Returns 191 ------- 192 corrected_index : int 193 the index in the full dataset 194 """ 195 196 return index 197 198 def count_classes( 199 self, frac: bool = False, zeros: bool = False, bouts: bool = False 200 ) -> Dict: 201 """ 202 Get a dictionary with class-wise frame counts 203 204 Parameters 205 ---------- 206 frac : bool, default False 207 if True, a fraction of the total frame count is returned 208 209 Returns 210 ------- 211 count_dictionary : dict 212 a dictionary with class indices as keys and frame counts as values 213 """ 214 215 return {} 216 217 def behaviors_dict(self) -> Dict: 218 """ 219 Get a dictionary of class names 220 221 Returns 222 ------- 223 behavior_dictionary: dict 224 a dictionary with class indices as keys and class names as values 225 """ 226 227 return {} 228 229 def annotation_class(self) -> str: 230 """ 231 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon) 232 233 Returns 234 ------- 235 annotation_class : str 236 the type of annotation 237 """ 238 239 return "none" 240 241 def size(self) -> int: 242 """ 243 Get the total number of frames in the data 244 245 Returns 246 ------- 247 size : int 248 the total number of frames 249 """ 250 251 return None 252 253 def filtered_indices(self) -> List: 254 """ 255 Return the indices of the samples that should be removed 256 257 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 258 parameters 259 260 Returns 261 ------- 262 indices_to_remove : list 263 a list of integer indices that should be removed 264 """ 265 266 return [] 267 268 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 269 """ 270 Set pseudo labels to the unlabeled data 271 272 Parameters 273 ---------- 274 labels : torch.Tensor 275 a tensor of pseudo-labels for the unlabeled data 276 """ 277 278 pass
A class that generates annotation from video information and stores it
Processes input video information and generates ordered arrays of annotation samples and corresponding unique original coordinates, as well as some meta objects. It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). The original coordinates object contains information about the video_id, clip_id and start time of the samples in the original input data. An AnnotationStore has to be fully defined with a tuple of key objects. The annotation array can be accessed with integer indices. The samples can be stored as a torch.Tensor in RAM or as an array of file paths to be loaded on runtime. When no arguments are passed a blank class instance should be created that can later be filled with information from key objects
29 def __init__( 30 self, video_order: List = None, key_objects: Tuple = None, *args, **kwargs 31 ): 32 """ 33 Parameters 34 ---------- 35 video_order : list, optional 36 a list of video ids that should be processed in the same order (not passed if creating from key objects) 37 key_objects : tuple, optional 38 a tuple of key objects 39 """ 40 41 pass
Parameters
video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) key_objects : tuple, optional a tuple of key objects
55 def remove(self, indices: List) -> None: 56 """ 57 Remove the samples corresponding to indices 58 59 Parameters 60 ---------- 61 indices : int 62 a list of integer indices to remove 63 """ 64 65 pass
Remove the samples corresponding to indices
Parameters
indices : int a list of integer indices to remove
67 def key_objects(self) -> Tuple: 68 """ 69 Return a tuple of the key objects necessary to re-create the Store 70 71 Returns 72 ------- 73 key_objects : tuple 74 a tuple of key objects 75 """ 76 77 return ()
Return a tuple of the key objects necessary to re-create the Store
Returns
key_objects : tuple a tuple of key objects
79 def load_from_key_objects(self, key_objects: Tuple) -> None: 80 """ 81 Load the information from a tuple of key objects 82 83 Parameters 84 ---------- 85 key_objects : tuple 86 a tuple of key objects 87 """ 88 89 pass
Load the information from a tuple of key objects
Parameters
key_objects : tuple a tuple of key objects
91 def to_ram(self) -> None: 92 """ 93 Transfer the data samples to RAM if they were previously stored as file paths 94 """ 95 96 pass
Transfer the data samples to RAM if they were previously stored as file paths
98 def get_original_coordinates(self) -> np.ndarray: 99 """ 100 Return the original coordinates array 101 102 Returns 103 ------- 104 np.ndarray 105 an array that contains the coordinates of the data samples in original input data (video id, clip id, 106 start frame) 107 """ 108 109 return None
Return the original coordinates array
Returns
np.ndarray an array that contains the coordinates of the data samples in original input data (video id, clip id, start frame)
111 def create_subsample(self, indices: List, ssl_indices: List = None): 112 """ 113 Create a new store that contains a subsample of the data 114 115 Parameters 116 ---------- 117 indices : list 118 the indices to be included in the subsample 119 ssl_indices : list, optional 120 the indices to be included in the subsample without the annotation data 121 """ 122 123 return self.new()
Create a new store that contains a subsample of the data
Parameters
indices : list the indices to be included in the subsample ssl_indices : list, optional the indices to be included in the subsample without the annotation data
125 @classmethod 126 def get_file_ids(cls, *args, **kwargs) -> List: 127 """ 128 Process data parameters and return a list of ids of the videos that should 129 be processed by the __init__ function 130 131 Returns 132 ------- 133 video_ids : list 134 a list of video file ids 135 """ 136 137 return None
Process data parameters and return a list of ids of the videos that should be processed by the __init__ function
Returns
video_ids : list a list of video file ids
156 def get_len(self, return_unlabeled: bool) -> int: 157 """ 158 Get the length of the subsample of labeled/unlabeled data 159 160 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 161 and if return_unlabeled is None the index is already correct 162 163 Parameters 164 ---------- 165 return_unlabeled : bool 166 the identifier for the subsample 167 168 Returns 169 ------- 170 length : int 171 the length of the subsample 172 """ 173 174 return None
Get the length of the subsample of labeled/unlabeled data
If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct
Parameters
return_unlabeled : bool the identifier for the subsample
Returns
length : int the length of the subsample
176 def get_idx(self, index: int, return_unlabeled: bool) -> int: 177 """ 178 Convert from an index in the subsample of labeled/unlabeled data to an index in the full array 179 180 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 181 and if return_unlabeled is None the index is already correct 182 183 Parameters 184 ---------- 185 index : int 186 the index in the subsample 187 return_unlabeled : bool 188 the identifier for the subsample 189 190 Returns 191 ------- 192 corrected_index : int 193 the index in the full dataset 194 """ 195 196 return index
Convert from an index in the subsample of labeled/unlabeled data to an index in the full array
If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct
Parameters
index : int the index in the subsample return_unlabeled : bool the identifier for the subsample
Returns
corrected_index : int the index in the full dataset
198 def count_classes( 199 self, frac: bool = False, zeros: bool = False, bouts: bool = False 200 ) -> Dict: 201 """ 202 Get a dictionary with class-wise frame counts 203 204 Parameters 205 ---------- 206 frac : bool, default False 207 if True, a fraction of the total frame count is returned 208 209 Returns 210 ------- 211 count_dictionary : dict 212 a dictionary with class indices as keys and frame counts as values 213 """ 214 215 return {}
Get a dictionary with class-wise frame counts
Parameters
frac : bool, default False if True, a fraction of the total frame count is returned
Returns
count_dictionary : dict a dictionary with class indices as keys and frame counts as values
217 def behaviors_dict(self) -> Dict: 218 """ 219 Get a dictionary of class names 220 221 Returns 222 ------- 223 behavior_dictionary: dict 224 a dictionary with class indices as keys and class names as values 225 """ 226 227 return {}
Get a dictionary of class names
Returns
behavior_dictionary: dict a dictionary with class indices as keys and class names as values
229 def annotation_class(self) -> str: 230 """ 231 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon) 232 233 Returns 234 ------- 235 annotation_class : str 236 the type of annotation 237 """ 238 239 return "none"
Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon)
Returns
annotation_class : str the type of annotation
241 def size(self) -> int: 242 """ 243 Get the total number of frames in the data 244 245 Returns 246 ------- 247 size : int 248 the total number of frames 249 """ 250 251 return None
Get the total number of frames in the data
Returns
size : int the total number of frames
253 def filtered_indices(self) -> List: 254 """ 255 Return the indices of the samples that should be removed 256 257 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 258 parameters 259 260 Returns 261 ------- 262 indices_to_remove : list 263 a list of integer indices that should be removed 264 """ 265 266 return []
Return the indices of the samples that should be removed
Choosing the indices can be based on any kind of filering defined in the __init__ function by the data parameters
Returns
indices_to_remove : list a list of integer indices that should be removed
268 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 269 """ 270 Set pseudo labels to the unlabeled data 271 272 Parameters 273 ---------- 274 labels : torch.Tensor 275 a tensor of pseudo-labels for the unlabeled data 276 """ 277 278 pass
Set pseudo labels to the unlabeled data
Parameters
labels : torch.Tensor a tensor of pseudo-labels for the unlabeled data
281class ActionSegmentationStore(AnnotationStore): # + 282 """ 283 A general realization of an annotation store for action segmentation tasks 284 285 Assumes the following file structure: 286 ``` 287 annotation_path 288 ├── video1_annotation.pickle 289 └── video2_labels.pickle 290 ``` 291 Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`. 292 """ 293 294 def __init__( 295 self, 296 video_order: List = None, 297 min_frames: Dict = None, 298 max_frames: Dict = None, 299 visibility: Dict = None, 300 exclusive: bool = True, 301 len_segment: int = 128, 302 overlap: int = 0, 303 behaviors: Set = None, 304 ignored_classes: Set = None, 305 ignored_clips: Set = None, 306 annotation_suffix: Union[Set, str] = None, 307 annotation_path: Union[Set, str] = None, 308 behavior_file: str = None, 309 correction: Dict = None, 310 frame_limit: int = 0, 311 filter_annotated: bool = False, 312 filter_background: bool = False, 313 error_class: str = None, 314 min_frames_action: int = None, 315 key_objects: Tuple = None, 316 visibility_min_score: float = 0.2, 317 visibility_min_frac: float = 0.7, 318 mask: Dict = None, 319 use_hard_negatives: bool = False, 320 interactive: bool = False, 321 *args, 322 **kwargs, 323 ) -> None: 324 """ 325 Parameters 326 ---------- 327 video_order : list, optional 328 a list of video ids that should be processed in the same order (not passed if creating from key objects) 329 min_frames : dict, optional 330 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 331 clip start frames (not passed if creating from key objects) 332 max_frames : dict, optional 333 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 334 clip end frames (not passed if creating from key objects) 335 visibility : dict, optional 336 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 337 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 338 exclusive : bool, default True 339 if True, the annotation is single-label; if False, multi-label 340 len_segment : int, default 128 341 the length of the segments in which the data should be cut (in frames) 342 overlap : int, default 0 343 the length of the overlap between neighboring segments (in frames) 344 behaviors : set, optional 345 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 346 loaded from a file) 347 ignored_classes : set, optional 348 the list of behaviors from the behaviors list or file to not annotate 349 ignored_clips : set, optional 350 clip ids to ignore 351 annotation_suffix : str | set, optional 352 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 353 (not passed if creating from key objects or if irrelevant for the dataset) 354 annotation_path : str | set, optional 355 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 356 from key objects) 357 behavior_file : str, optional 358 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 359 correction : dict, optional 360 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 361 can be used to correct for variations in naming or to merge several labels in one 362 frame_limit : int, default 0 363 the smallest possible length of a clip (shorter clips are discarded) 364 filter_annotated : bool, default False 365 if True, the samples that do not have any labels will be filtered 366 filter_background : bool, default False 367 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 368 error_class : str, optional 369 the name of the error class (the annotations that intersect with this label will be discarded) 370 min_frames_action : int, default 0 371 the minimum length of an action (shorter actions are not annotated) 372 key_objects : tuple, optional 373 the key objects to load the AnnotationStore from 374 visibility_min_score : float, default 5 375 the minimum visibility score for visibility filtering 376 visibility_min_frac : float, default 0.7 377 the minimum fraction of visible frames for visibility filtering 378 mask : dict, optional 379 a masked value dictionary (for active learning simulation experiments) 380 use_hard_negatives : bool, default False 381 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 382 interactive : bool, default False 383 if `True`, annotation is assigned to pairs of individuals 384 """ 385 386 super().__init__() 387 388 if ignored_clips is None: 389 ignored_clips = [] 390 self.len_segment = int(len_segment) 391 self.exclusive = exclusive 392 if overlap < 1: 393 overlap = overlap * len_segment 394 self.overlap = int(overlap) 395 self.video_order = video_order 396 self.min_frames = min_frames 397 self.max_frames = max_frames 398 self.visibility = visibility 399 self.vis_min_score = visibility_min_score 400 self.vis_min_frac = visibility_min_frac 401 self.mask = mask 402 self.use_negatives = use_hard_negatives 403 self.interactive = interactive 404 self.ignored_clips = ignored_clips 405 self.file_paths = self._get_file_paths(annotation_path) 406 self.ignored_classes = ignored_classes 407 self.update_behaviors = False 408 409 self.ram = True 410 self.original_coordinates = [] 411 self.filtered = [] 412 413 self.step = self.len_segment - self.overlap 414 415 self.ann_suffix = annotation_suffix 416 self.annotation_folder = annotation_path 417 self.filter_annotated = filter_annotated 418 self.filter_background = filter_background 419 self.frame_limit = frame_limit 420 self.min_frames_action = min_frames_action 421 self.error_class = error_class 422 423 if correction is None: 424 correction = {} 425 self.correction = correction 426 427 if self.max_frames is None: 428 self.max_frames = defaultdict(lambda: {}) 429 if self.min_frames is None: 430 self.min_frames = defaultdict(lambda: {}) 431 432 lists = [self.annotation_folder, self.ann_suffix] 433 for i in range(len(lists)): 434 iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str)) 435 if lists[i] is not None: 436 if not iterable: 437 lists[i] = [lists[i]] 438 lists[i] = [x for x in lists[i]] 439 self.annotation_folder, self.ann_suffix = lists 440 441 if ignored_classes is None: 442 ignored_classes = [] 443 self.ignored_classes = ignored_classes 444 self._set_behaviors(behaviors, ignored_classes, behavior_file) 445 446 if key_objects is None and self.video_order is not None: 447 self.data = self._load_data() 448 elif key_objects is not None: 449 self.load_from_key_objects(key_objects) 450 else: 451 self.data = None 452 self.labeled_indices, self.unlabeled_indices = self._compute_labeled() 453 454 def __getitem__(self, ind): 455 if self.data is None: 456 raise RuntimeError("The annotation store data has not been initialized!") 457 return self.data[ind] 458 459 def __len__(self) -> int: 460 if self.data is None: 461 raise RuntimeError("The annotation store data has not been initialized!") 462 return len(self.data) 463 464 def remove(self, indices: List) -> None: 465 """ 466 Remove the samples corresponding to indices 467 468 Parameters 469 ---------- 470 indices : list 471 a list of integer indices to remove 472 """ 473 474 if len(indices) > 0: 475 mask = np.ones(len(self.data)) 476 mask[indices] = 0 477 mask = mask.astype(bool) 478 self.data = self.data[mask] 479 self.original_coordinates = self.original_coordinates[mask] 480 481 def key_objects(self) -> Tuple: 482 """ 483 Return a tuple of the key objects necessary to re-create the Store 484 485 Returns 486 ------- 487 key_objects : tuple 488 a tuple of key objects 489 """ 490 491 return ( 492 self.original_coordinates, 493 self.data, 494 self.behaviors, 495 self.exclusive, 496 self.len_segment, 497 self.step, 498 self.overlap, 499 ) 500 501 def load_from_key_objects(self, key_objects: Tuple) -> None: 502 """ 503 Load the information from a tuple of key objects 504 505 Parameters 506 ---------- 507 key_objects : tuple 508 a tuple of key objects 509 """ 510 511 ( 512 self.original_coordinates, 513 self.data, 514 self.behaviors, 515 self.exclusive, 516 self.len_segment, 517 self.step, 518 self.overlap, 519 ) = key_objects 520 self.labeled_indices, self.unlabeled_indices = self._compute_labeled() 521 522 def to_ram(self) -> None: 523 """ 524 Transfer the data samples to RAM if they were previously stored as file paths 525 """ 526 527 pass 528 529 def get_original_coordinates(self) -> np.ndarray: 530 """ 531 Return the video_indices array 532 533 Returns 534 ------- 535 original_coordinates : numpy.ndarray 536 an array that contains the coordinates of the data samples in original input data 537 """ 538 return self.original_coordinates 539 540 def create_subsample(self, indices: List, ssl_indices: List = None): 541 """ 542 Create a new store that contains a subsample of the data 543 544 Parameters 545 ---------- 546 indices : list 547 the indices to be included in the subsample 548 ssl_indices : list, optional 549 the indices to be included in the subsample without the annotation data 550 """ 551 552 if ssl_indices is None: 553 ssl_indices = [] 554 data = copy(self.data) 555 data[ssl_indices, ...] = -100 556 new = self.new() 557 new.original_coordinates = self.original_coordinates[indices + ssl_indices] 558 new.data = self.data[indices + ssl_indices] 559 new.labeled_indices, new.unlabeled_indices = new._compute_labeled() 560 new.behaviors = self.behaviors 561 new.exclusive = self.exclusive 562 new.len_segment = self.len_segment 563 new.step = self.step 564 new.overlap = self.overlap 565 new.max_frames = self.max_frames 566 new.min_frames = self.min_frames 567 return new 568 569 def get_len(self, return_unlabeled: bool) -> int: 570 """ 571 Get the length of the subsample of labeled/unlabeled data 572 573 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 574 and if return_unlabeled is None the index is already correct 575 576 Parameters 577 ---------- 578 return_unlabeled : bool 579 the identifier for the subsample 580 581 Returns 582 ------- 583 length : int 584 the length of the subsample 585 """ 586 587 if self.data is None: 588 raise RuntimeError("The annotation store data has not been initialized!") 589 elif return_unlabeled is None: 590 return len(self.data) 591 elif return_unlabeled: 592 return len(self.unlabeled_indices) 593 else: 594 return len(self.labeled_indices) 595 596 def get_indices(self, return_unlabeled: bool) -> List: 597 """ 598 Get a list of indices of samples in the labeled/unlabeled subset 599 600 Parameters 601 ---------- 602 return_unlabeled : bool 603 the identifier for the subsample (`True` for unlabeled, `False` for labeled, `None` for the 604 whole dataset) 605 606 Returns 607 ------- 608 indices : list 609 a list of indices that meet the criteria 610 """ 611 612 return list(range(len(self.data))) 613 614 def count_classes( 615 self, perc: bool = False, zeros: bool = False, bouts: bool = False 616 ) -> Dict: 617 """ 618 Get a dictionary with class-wise frame counts 619 620 Parameters 621 ---------- 622 perc : bool, default False 623 if `True`, a fraction of the total frame count is returned 624 zeros : bool, default False 625 if `True` and annotation is not exclusive, zero counts are returned 626 bouts : bool, default False 627 if `True`, instead of frame counts segment counts are returned 628 629 Returns 630 ------- 631 count_dictionary : dict 632 a dictionary with class indices as keys and frame counts as values 633 """ 634 635 if bouts: 636 if self.overlap != 0: 637 data = {} 638 for video, value in self.max_frames.items(): 639 for clip, end in value.items(): 640 length = end - self._get_min_frame(video, clip) 641 if self.exclusive: 642 data[f"{video}---{clip}"] = -100 * torch.ones(length) 643 else: 644 data[f"{video}---{clip}"] = -100 * torch.ones( 645 (len(self.behaviors_dict()), length) 646 ) 647 for x, coords in zip(self.data, self.original_coordinates): 648 split = coords[0].split("---") 649 l = self._get_max_frame(split[0], split[1]) - self._get_min_frame( 650 split[0], split[1] 651 ) 652 i = coords[1] 653 start = int(i) * self.step 654 end = min(start + self.len_segment, l) 655 data[coords[0]][..., start:end] = x[..., : end - start] 656 values = [] 657 for key, value in data.items(): 658 values.append(value) 659 values.append(-100 * torch.ones((*value.shape[:-1], 1))) 660 data = torch.cat(values, -1).T 661 else: 662 data = copy(self.data) 663 if self.exclusive: 664 data = data.flatten() 665 else: 666 data = data.transpose(1, 2).reshape(-1, len(self.behaviors)) 667 count_dictionary = {} 668 for c in self.behaviors_dict(): 669 if self.exclusive: 670 arr = data == c 671 else: 672 if zeros: 673 arr = data[:, c] == 0 674 else: 675 arr = data[:, c] == 1 676 output, indices = torch.unique_consecutive(arr, return_inverse=True) 677 true_indices = torch.where(output)[0] 678 count_dictionary[c] = len(true_indices) 679 else: 680 ind = 1 681 if zeros: 682 ind = 0 683 if self.exclusive: 684 count_dictionary = dict(Counter(self.data.flatten().cpu().numpy())) 685 else: 686 d = {} 687 for i in range(self.data.shape[1]): 688 cnt = Counter(self.data[:, i, :].flatten().cpu().numpy()) 689 d[i] = cnt[ind] 690 count_dictionary = d 691 if perc: 692 total = sum([v for k, v in count_dictionary.items()]) 693 count_dictionary = {k: v / total for k, v in count_dictionary.items()} 694 for i in self.behaviors_dict(): 695 if i not in count_dictionary: 696 count_dictionary[i] = 0 697 return count_dictionary 698 699 def behaviors_dict(self) -> Dict: 700 """ 701 Get a dictionary of class names 702 703 Returns 704 ------- 705 behavior_dictionary: dict 706 a dictionary with class indices as keys and class names as values 707 """ 708 709 # if self.behaviors is None 710 if self.exclusive and "other" not in self.behaviors: 711 d = {i + 1: b for i, b in enumerate(self.behaviors)} 712 d[0] = "other" 713 else: 714 d = {i: b for i, b in enumerate(self.behaviors)} 715 return d 716 717 def annotation_class(self) -> str: 718 """ 719 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification') 720 721 Returns 722 ------- 723 annotation_class : str 724 the type of annotation 725 """ 726 727 if self.exclusive: 728 return "exclusive_classification" 729 else: 730 return "nonexclusive_classification" 731 732 def size(self) -> int: 733 """ 734 Get the total number of frames in the data 735 736 Returns 737 ------- 738 size : int 739 the total number of frames 740 """ 741 742 return self.data.shape[0] * self.data.shape[-1] 743 744 def filtered_indices(self) -> List: 745 """ 746 Return the indices of the samples that should be removed 747 748 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 749 parameters 750 751 Returns 752 ------- 753 indices_to_remove : list 754 a list of integer indices that should be removed 755 """ 756 757 return self.filtered 758 759 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 760 """ 761 Set pseudo labels to the unlabeled data 762 763 Parameters 764 ---------- 765 labels : torch.Tensor 766 a tensor of pseudo-labels for the unlabeled data 767 """ 768 769 self.data[self.unlabeled_indices] = labels 770 771 @classmethod 772 def get_file_ids( 773 cls, 774 annotation_path: Union[str, Set], 775 annotation_suffix: Union[str, Set], 776 *args, 777 **kwargs, 778 ) -> List: 779 """ 780 Process data parameters and return a list of ids of the videos that should 781 be processed by the __init__ function 782 783 Parameters 784 ---------- 785 annotation_path : str | set 786 the path or the set of paths to the folder where the annotation files are stored 787 annotation_suffix : str | set, optional 788 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 789 790 Returns 791 ------- 792 video_ids : list 793 a list of video file ids 794 """ 795 796 lists = [annotation_path, annotation_suffix] 797 for i in range(len(lists)): 798 iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str)) 799 if lists[i] is not None: 800 if not iterable: 801 lists[i] = [lists[i]] 802 lists[i] = [x for x in lists[i]] 803 annotation_path, annotation_suffix = lists 804 files = [] 805 for folder in annotation_path: 806 files += [ 807 strip_suffix(os.path.basename(file), annotation_suffix) 808 for file in os.listdir(folder) 809 if file.endswith(tuple([x for x in annotation_suffix])) 810 ] 811 files = sorted(files, key=lambda x: os.path.basename(x)) 812 return files 813 814 def _set_behaviors( 815 self, behaviors: List, ignored_classes: List, behavior_file: str 816 ): 817 """ 818 Get a list of behaviors that should be annotated from behavior parameters 819 """ 820 821 if behaviors is not None: 822 for b in ignored_classes: 823 if b in behaviors: 824 behaviors.remove(b) 825 self.behaviors = behaviors 826 827 def _compute_labeled(self) -> Tuple[torch.Tensor, torch.Tensor]: 828 """ 829 Get the indices of labeled (annotated) and unlabeled samples 830 """ 831 832 if self.data is not None and len(self.data) > 0: 833 unlabeled = torch.sum(self.data != -100, dim=1) == 0 834 labeled_indices = torch.where(~unlabeled)[0] 835 unlabeled_indices = torch.where(unlabeled)[0] 836 else: 837 labeled_indices, unlabeled_indices = torch.tensor([]), torch.tensor([]) 838 return labeled_indices, unlabeled_indices 839 840 def _generate_annotation(self, times: Dict, name: str) -> Dict: 841 """ 842 Process a loaded annotation file to generate a training labels dictionary 843 """ 844 845 annotation = {} 846 if self.behaviors is None and times is not None: 847 behaviors = set() 848 for d in times.values(): 849 behaviors.update([k for k, v in d.items()]) 850 self.behaviors = [ 851 x 852 for x in sorted(behaviors) 853 if x not in self.ignored_classes 854 and not x.startswith("negative") 855 and not x.startswith("unknown") 856 ] 857 elif self.behaviors is None and times is None: 858 raise ValueError("Cannot generate annotqtion without behavior information!") 859 beh_inv = {v: k for k, v in self.behaviors_dict().items()} 860 # if there is no annotation file, generate empty annotation 861 if self.interactive: 862 clips = [ 863 "+".join(sorted(x)) 864 for x in combinations(self.max_frames[name].keys(), 2) 865 ] 866 else: 867 clips = list(self.max_frames[name].keys()) 868 if times is None: 869 clips = [x for x in clips if x not in self.ignored_clips] 870 # otherwise, apply filters and generate label arrays 871 else: 872 clips = [ 873 x 874 for x in clips 875 if x not in self.ignored_clips and x not in times.keys() 876 ] 877 for ind in times.keys(): 878 try: 879 min_frame = self._get_min_frame(name, ind) 880 max_frame = self._get_max_frame(name, ind) 881 except KeyError: 882 continue 883 go_on = max_frame - min_frame + 1 >= self.frame_limit 884 if go_on: 885 v_len = max_frame - min_frame + 1 886 if self.exclusive: 887 if not self.filter_background: 888 value = beh_inv.get("other", 0) 889 labels = np.ones(v_len, dtype=np.compat.long) * value 890 else: 891 labels = -100 * np.ones(v_len, dtype=np.compat.long) 892 else: 893 labels = np.zeros((len(self.behaviors), v_len), dtype=np.float) 894 cat_new = [] 895 for cat in times[ind].keys(): 896 if cat.startswith("unknown"): 897 cat_new.append(cat) 898 for cat in times[ind].keys(): 899 if cat.startswith("negative"): 900 cat_new.append(cat) 901 for cat in times[ind].keys(): 902 if not cat.startswith("negative") and not cat.startswith( 903 "unknown" 904 ): 905 cat_new.append(cat) 906 for cat in cat_new: 907 neg = False 908 unknown = False 909 cat_times = times[ind][cat] 910 if self.use_negatives and cat.startswith("negative"): 911 cat = " ".join(cat.split()[1:]) 912 neg = True 913 elif cat.startswith("unknown"): 914 cat = " ".join(cat.split()[1:]) 915 unknown = True 916 if cat in self.correction: 917 cat = self.correction[cat] 918 for start, end, amb in cat_times: 919 if end > self._get_max_frame(name, ind) + 1: 920 end = self._get_max_frame(name, ind) + 1 921 if amb != 0: 922 continue 923 start -= min_frame 924 end -= min_frame 925 if ( 926 self.min_frames_action is not None 927 and end - start < self.min_frames_action 928 ): 929 continue 930 if ( 931 self.vis_min_frac > 0 932 and self.vis_min_score > 0 933 and self.visibility is not None 934 ): 935 s = 0 936 for ind_k in ind.split("+"): 937 s += np.sum( 938 self.visibility[name][ind_k][start:end] 939 > self.vis_min_score 940 ) 941 if s < self.vis_min_frac * (end - start) * len( 942 ind.split("+") 943 ): 944 continue 945 if cat in beh_inv: 946 cat_i_global = beh_inv[cat] 947 if self.exclusive: 948 labels[start:end] = cat_i_global 949 else: 950 if unknown: 951 labels[cat_i_global, start:end] = -100 952 elif neg: 953 labels[cat_i_global, start:end] = 2 954 else: 955 labels[cat_i_global, start:end] = 1 956 else: 957 self.not_found.add(cat) 958 if self.filter_background: 959 if not self.exclusive: 960 labels[:, start:end][ 961 labels[:, start:end] == 0 962 ] = 3 963 else: 964 labels[start:end][labels[start:end] == -100] = 0 965 966 if self.error_class is not None and self.error_class in times[ind]: 967 for start, end, amb in times[ind][self.error_class]: 968 if self.exclusive: 969 labels[start:end] = -100 970 else: 971 labels[:, start:end] = -100 972 annotation[os.path.basename(name) + "---" + str(ind)] = labels 973 for ind in clips: 974 try: 975 min_frame = self._get_min_frame(name, ind) 976 max_frame = self._get_max_frame(name, ind) 977 except KeyError: 978 continue 979 go_on = max_frame - min_frame + 1 >= self.frame_limit 980 if go_on: 981 v_len = max_frame - min_frame + 1 982 if self.exclusive: 983 annotation[ 984 os.path.basename(name) + "---" + str(ind) 985 ] = -100 * np.ones(v_len, dtype=np.compat.long) 986 else: 987 annotation[ 988 os.path.basename(name) + "---" + str(ind) 989 ] = -100 * np.ones((len(self.behaviors), v_len), dtype=np.float) 990 return annotation 991 992 def _make_trimmed_annotations(self, annotations_dict: Dict) -> torch.Tensor: 993 """ 994 Cut a label dictionary into overlapping pieces of equal length 995 """ 996 997 labels = [] 998 self.original_coordinates = [] 999 masked_all = [] 1000 for v_id in sorted(annotations_dict.keys()): 1001 if v_id in annotations_dict: 1002 annotations = annotations_dict[v_id] 1003 else: 1004 raise ValueError( 1005 f'The id list in {v_id.split("---")[0]} is not consistent across files' 1006 ) 1007 split = v_id.split("---") 1008 if len(split) > 1: 1009 video_id, ind = split 1010 else: 1011 video_id = split[0] 1012 ind = "" 1013 min_frame = self._get_min_frame(video_id, ind) 1014 max_frame = self._get_max_frame(video_id, ind) 1015 v_len = max_frame - min_frame + 1 1016 sp = np.arange(0, v_len, self.step) 1017 pad = sp[-1] + self.len_segment - v_len 1018 if self.exclusive: 1019 annotations = np.pad(annotations, ((0, pad)), constant_values=-100) 1020 else: 1021 annotations = np.pad( 1022 annotations, ((0, 0), (0, pad)), constant_values=-100 1023 ) 1024 masked = np.zeros(annotations.shape) 1025 if ( 1026 self.mask is not None 1027 and video_id in self.mask["masked"] 1028 and ind in self.mask["masked"][video_id] 1029 ): 1030 for start, end in self.mask["masked"][video_id][ind]: 1031 masked[..., int(start) : int(end)] = 1 1032 for i, start in enumerate(sp): 1033 self.original_coordinates.append((v_id, i)) 1034 if self.exclusive: 1035 ann = annotations[start : start + self.len_segment] 1036 m = masked[start : start + self.len_segment] 1037 else: 1038 ann = annotations[:, start : start + self.len_segment] 1039 m = masked[:, start : start + self.len_segment] 1040 labels.append(ann) 1041 masked_all.append(m) 1042 self.original_coordinates = np.array(self.original_coordinates) 1043 labels = torch.tensor(np.array(labels)) 1044 masked_all = torch.tensor(np.array(masked_all)).int().bool() 1045 if self.filter_background and not self.exclusive: 1046 for i, label in enumerate(labels): 1047 label[:, torch.sum((label == 1) | (label == 3), 0) == 0] = -100 1048 label[label == 3] = 0 1049 labels[(labels != -100) & masked_all] = -200 1050 return labels 1051 1052 @classmethod 1053 def _get_file_paths(cls, annotation_path: Union[str, Set]) -> List: 1054 """ 1055 Get a list of relevant files 1056 """ 1057 1058 file_paths = [] 1059 if annotation_path is not None: 1060 if isinstance(annotation_path, str): 1061 annotation_path = [annotation_path] 1062 for folder in annotation_path: 1063 file_paths += [os.path.join(folder, x) for x in os.listdir(folder)] 1064 return file_paths 1065 1066 def _get_max_frame(self, video_id: str, clip_id: str): 1067 """ 1068 Get the end frame of a clip in a video 1069 """ 1070 1071 if clip_id in self.max_frames[video_id]: 1072 return self.max_frames[video_id][clip_id] 1073 else: 1074 return min( 1075 [self.max_frames[video_id][ind_k] for ind_k in clip_id.split("+")] 1076 ) 1077 1078 def _get_min_frame(self, video_id, clip_id): 1079 """ 1080 Get the start frame of a clip in a video 1081 """ 1082 1083 if clip_id in self.min_frames[video_id]: 1084 return self.min_frames[video_id][clip_id] 1085 else: 1086 return max( 1087 [self.min_frames[video_id][ind_k] for ind_k in clip_id.split("+")] 1088 ) 1089 1090 @abstractmethod 1091 def _load_data(self) -> torch.Tensor: 1092 """ 1093 Load behavior annotation and generate annotation prompts 1094 """
A general realization of an annotation store for action segmentation tasks
Assumes the following file structure:
annotation_path
├── video1_annotation.pickle
└── video2_labels.pickle
Here annotation_suffix
is {'_annotation.pickle', '_labels.pickle'}
.
294 def __init__( 295 self, 296 video_order: List = None, 297 min_frames: Dict = None, 298 max_frames: Dict = None, 299 visibility: Dict = None, 300 exclusive: bool = True, 301 len_segment: int = 128, 302 overlap: int = 0, 303 behaviors: Set = None, 304 ignored_classes: Set = None, 305 ignored_clips: Set = None, 306 annotation_suffix: Union[Set, str] = None, 307 annotation_path: Union[Set, str] = None, 308 behavior_file: str = None, 309 correction: Dict = None, 310 frame_limit: int = 0, 311 filter_annotated: bool = False, 312 filter_background: bool = False, 313 error_class: str = None, 314 min_frames_action: int = None, 315 key_objects: Tuple = None, 316 visibility_min_score: float = 0.2, 317 visibility_min_frac: float = 0.7, 318 mask: Dict = None, 319 use_hard_negatives: bool = False, 320 interactive: bool = False, 321 *args, 322 **kwargs, 323 ) -> None: 324 """ 325 Parameters 326 ---------- 327 video_order : list, optional 328 a list of video ids that should be processed in the same order (not passed if creating from key objects) 329 min_frames : dict, optional 330 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 331 clip start frames (not passed if creating from key objects) 332 max_frames : dict, optional 333 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 334 clip end frames (not passed if creating from key objects) 335 visibility : dict, optional 336 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 337 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 338 exclusive : bool, default True 339 if True, the annotation is single-label; if False, multi-label 340 len_segment : int, default 128 341 the length of the segments in which the data should be cut (in frames) 342 overlap : int, default 0 343 the length of the overlap between neighboring segments (in frames) 344 behaviors : set, optional 345 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 346 loaded from a file) 347 ignored_classes : set, optional 348 the list of behaviors from the behaviors list or file to not annotate 349 ignored_clips : set, optional 350 clip ids to ignore 351 annotation_suffix : str | set, optional 352 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 353 (not passed if creating from key objects or if irrelevant for the dataset) 354 annotation_path : str | set, optional 355 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 356 from key objects) 357 behavior_file : str, optional 358 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 359 correction : dict, optional 360 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 361 can be used to correct for variations in naming or to merge several labels in one 362 frame_limit : int, default 0 363 the smallest possible length of a clip (shorter clips are discarded) 364 filter_annotated : bool, default False 365 if True, the samples that do not have any labels will be filtered 366 filter_background : bool, default False 367 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 368 error_class : str, optional 369 the name of the error class (the annotations that intersect with this label will be discarded) 370 min_frames_action : int, default 0 371 the minimum length of an action (shorter actions are not annotated) 372 key_objects : tuple, optional 373 the key objects to load the AnnotationStore from 374 visibility_min_score : float, default 5 375 the minimum visibility score for visibility filtering 376 visibility_min_frac : float, default 0.7 377 the minimum fraction of visible frames for visibility filtering 378 mask : dict, optional 379 a masked value dictionary (for active learning simulation experiments) 380 use_hard_negatives : bool, default False 381 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 382 interactive : bool, default False 383 if `True`, annotation is assigned to pairs of individuals 384 """ 385 386 super().__init__() 387 388 if ignored_clips is None: 389 ignored_clips = [] 390 self.len_segment = int(len_segment) 391 self.exclusive = exclusive 392 if overlap < 1: 393 overlap = overlap * len_segment 394 self.overlap = int(overlap) 395 self.video_order = video_order 396 self.min_frames = min_frames 397 self.max_frames = max_frames 398 self.visibility = visibility 399 self.vis_min_score = visibility_min_score 400 self.vis_min_frac = visibility_min_frac 401 self.mask = mask 402 self.use_negatives = use_hard_negatives 403 self.interactive = interactive 404 self.ignored_clips = ignored_clips 405 self.file_paths = self._get_file_paths(annotation_path) 406 self.ignored_classes = ignored_classes 407 self.update_behaviors = False 408 409 self.ram = True 410 self.original_coordinates = [] 411 self.filtered = [] 412 413 self.step = self.len_segment - self.overlap 414 415 self.ann_suffix = annotation_suffix 416 self.annotation_folder = annotation_path 417 self.filter_annotated = filter_annotated 418 self.filter_background = filter_background 419 self.frame_limit = frame_limit 420 self.min_frames_action = min_frames_action 421 self.error_class = error_class 422 423 if correction is None: 424 correction = {} 425 self.correction = correction 426 427 if self.max_frames is None: 428 self.max_frames = defaultdict(lambda: {}) 429 if self.min_frames is None: 430 self.min_frames = defaultdict(lambda: {}) 431 432 lists = [self.annotation_folder, self.ann_suffix] 433 for i in range(len(lists)): 434 iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str)) 435 if lists[i] is not None: 436 if not iterable: 437 lists[i] = [lists[i]] 438 lists[i] = [x for x in lists[i]] 439 self.annotation_folder, self.ann_suffix = lists 440 441 if ignored_classes is None: 442 ignored_classes = [] 443 self.ignored_classes = ignored_classes 444 self._set_behaviors(behaviors, ignored_classes, behavior_file) 445 446 if key_objects is None and self.video_order is not None: 447 self.data = self._load_data() 448 elif key_objects is not None: 449 self.load_from_key_objects(key_objects) 450 else: 451 self.data = None 452 self.labeled_indices, self.unlabeled_indices = self._compute_labeled()
Parameters
video_order : list, optional
a list of video ids that should be processed in the same order (not passed if creating from key objects)
min_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip start frames (not passed if creating from key objects)
max_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip end frames (not passed if creating from key objects)
visibility : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
exclusive : bool, default True
if True, the annotation is single-label; if False, multi-label
len_segment : int, default 128
the length of the segments in which the data should be cut (in frames)
overlap : int, default 0
the length of the overlap between neighboring segments (in frames)
behaviors : set, optional
the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
loaded from a file)
ignored_classes : set, optional
the list of behaviors from the behaviors list or file to not annotate
ignored_clips : set, optional
clip ids to ignore
annotation_suffix : str | set, optional
the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
(not passed if creating from key objects or if irrelevant for the dataset)
annotation_path : str | set, optional
the path or the set of paths to the folder where the annotation files are stored (not passed if creating
from key objects)
behavior_file : str, optional
the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
correction : dict, optional
a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
can be used to correct for variations in naming or to merge several labels in one
frame_limit : int, default 0
the smallest possible length of a clip (shorter clips are discarded)
filter_annotated : bool, default False
if True, the samples that do not have any labels will be filtered
filter_background : bool, default False
if True, only the unlabeled frames that are close to annotated frames will be labeled as background
error_class : str, optional
the name of the error class (the annotations that intersect with this label will be discarded)
min_frames_action : int, default 0
the minimum length of an action (shorter actions are not annotated)
key_objects : tuple, optional
the key objects to load the AnnotationStore from
visibility_min_score : float, default 5
the minimum visibility score for visibility filtering
visibility_min_frac : float, default 0.7
the minimum fraction of visible frames for visibility filtering
mask : dict, optional
a masked value dictionary (for active learning simulation experiments)
use_hard_negatives : bool, default False
mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
interactive : bool, default False
if True
, annotation is assigned to pairs of individuals
464 def remove(self, indices: List) -> None: 465 """ 466 Remove the samples corresponding to indices 467 468 Parameters 469 ---------- 470 indices : list 471 a list of integer indices to remove 472 """ 473 474 if len(indices) > 0: 475 mask = np.ones(len(self.data)) 476 mask[indices] = 0 477 mask = mask.astype(bool) 478 self.data = self.data[mask] 479 self.original_coordinates = self.original_coordinates[mask]
Remove the samples corresponding to indices
Parameters
indices : list a list of integer indices to remove
481 def key_objects(self) -> Tuple: 482 """ 483 Return a tuple of the key objects necessary to re-create the Store 484 485 Returns 486 ------- 487 key_objects : tuple 488 a tuple of key objects 489 """ 490 491 return ( 492 self.original_coordinates, 493 self.data, 494 self.behaviors, 495 self.exclusive, 496 self.len_segment, 497 self.step, 498 self.overlap, 499 )
Return a tuple of the key objects necessary to re-create the Store
Returns
key_objects : tuple a tuple of key objects
501 def load_from_key_objects(self, key_objects: Tuple) -> None: 502 """ 503 Load the information from a tuple of key objects 504 505 Parameters 506 ---------- 507 key_objects : tuple 508 a tuple of key objects 509 """ 510 511 ( 512 self.original_coordinates, 513 self.data, 514 self.behaviors, 515 self.exclusive, 516 self.len_segment, 517 self.step, 518 self.overlap, 519 ) = key_objects 520 self.labeled_indices, self.unlabeled_indices = self._compute_labeled()
Load the information from a tuple of key objects
Parameters
key_objects : tuple a tuple of key objects
522 def to_ram(self) -> None: 523 """ 524 Transfer the data samples to RAM if they were previously stored as file paths 525 """ 526 527 pass
Transfer the data samples to RAM if they were previously stored as file paths
529 def get_original_coordinates(self) -> np.ndarray: 530 """ 531 Return the video_indices array 532 533 Returns 534 ------- 535 original_coordinates : numpy.ndarray 536 an array that contains the coordinates of the data samples in original input data 537 """ 538 return self.original_coordinates
Return the video_indices array
Returns
original_coordinates : numpy.ndarray an array that contains the coordinates of the data samples in original input data
540 def create_subsample(self, indices: List, ssl_indices: List = None): 541 """ 542 Create a new store that contains a subsample of the data 543 544 Parameters 545 ---------- 546 indices : list 547 the indices to be included in the subsample 548 ssl_indices : list, optional 549 the indices to be included in the subsample without the annotation data 550 """ 551 552 if ssl_indices is None: 553 ssl_indices = [] 554 data = copy(self.data) 555 data[ssl_indices, ...] = -100 556 new = self.new() 557 new.original_coordinates = self.original_coordinates[indices + ssl_indices] 558 new.data = self.data[indices + ssl_indices] 559 new.labeled_indices, new.unlabeled_indices = new._compute_labeled() 560 new.behaviors = self.behaviors 561 new.exclusive = self.exclusive 562 new.len_segment = self.len_segment 563 new.step = self.step 564 new.overlap = self.overlap 565 new.max_frames = self.max_frames 566 new.min_frames = self.min_frames 567 return new
Create a new store that contains a subsample of the data
Parameters
indices : list the indices to be included in the subsample ssl_indices : list, optional the indices to be included in the subsample without the annotation data
569 def get_len(self, return_unlabeled: bool) -> int: 570 """ 571 Get the length of the subsample of labeled/unlabeled data 572 573 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 574 and if return_unlabeled is None the index is already correct 575 576 Parameters 577 ---------- 578 return_unlabeled : bool 579 the identifier for the subsample 580 581 Returns 582 ------- 583 length : int 584 the length of the subsample 585 """ 586 587 if self.data is None: 588 raise RuntimeError("The annotation store data has not been initialized!") 589 elif return_unlabeled is None: 590 return len(self.data) 591 elif return_unlabeled: 592 return len(self.unlabeled_indices) 593 else: 594 return len(self.labeled_indices)
Get the length of the subsample of labeled/unlabeled data
If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct
Parameters
return_unlabeled : bool the identifier for the subsample
Returns
length : int the length of the subsample
596 def get_indices(self, return_unlabeled: bool) -> List: 597 """ 598 Get a list of indices of samples in the labeled/unlabeled subset 599 600 Parameters 601 ---------- 602 return_unlabeled : bool 603 the identifier for the subsample (`True` for unlabeled, `False` for labeled, `None` for the 604 whole dataset) 605 606 Returns 607 ------- 608 indices : list 609 a list of indices that meet the criteria 610 """ 611 612 return list(range(len(self.data)))
Get a list of indices of samples in the labeled/unlabeled subset
Parameters
return_unlabeled : bool
the identifier for the subsample (True
for unlabeled, False
for labeled, None
for the
whole dataset)
Returns
indices : list a list of indices that meet the criteria
614 def count_classes( 615 self, perc: bool = False, zeros: bool = False, bouts: bool = False 616 ) -> Dict: 617 """ 618 Get a dictionary with class-wise frame counts 619 620 Parameters 621 ---------- 622 perc : bool, default False 623 if `True`, a fraction of the total frame count is returned 624 zeros : bool, default False 625 if `True` and annotation is not exclusive, zero counts are returned 626 bouts : bool, default False 627 if `True`, instead of frame counts segment counts are returned 628 629 Returns 630 ------- 631 count_dictionary : dict 632 a dictionary with class indices as keys and frame counts as values 633 """ 634 635 if bouts: 636 if self.overlap != 0: 637 data = {} 638 for video, value in self.max_frames.items(): 639 for clip, end in value.items(): 640 length = end - self._get_min_frame(video, clip) 641 if self.exclusive: 642 data[f"{video}---{clip}"] = -100 * torch.ones(length) 643 else: 644 data[f"{video}---{clip}"] = -100 * torch.ones( 645 (len(self.behaviors_dict()), length) 646 ) 647 for x, coords in zip(self.data, self.original_coordinates): 648 split = coords[0].split("---") 649 l = self._get_max_frame(split[0], split[1]) - self._get_min_frame( 650 split[0], split[1] 651 ) 652 i = coords[1] 653 start = int(i) * self.step 654 end = min(start + self.len_segment, l) 655 data[coords[0]][..., start:end] = x[..., : end - start] 656 values = [] 657 for key, value in data.items(): 658 values.append(value) 659 values.append(-100 * torch.ones((*value.shape[:-1], 1))) 660 data = torch.cat(values, -1).T 661 else: 662 data = copy(self.data) 663 if self.exclusive: 664 data = data.flatten() 665 else: 666 data = data.transpose(1, 2).reshape(-1, len(self.behaviors)) 667 count_dictionary = {} 668 for c in self.behaviors_dict(): 669 if self.exclusive: 670 arr = data == c 671 else: 672 if zeros: 673 arr = data[:, c] == 0 674 else: 675 arr = data[:, c] == 1 676 output, indices = torch.unique_consecutive(arr, return_inverse=True) 677 true_indices = torch.where(output)[0] 678 count_dictionary[c] = len(true_indices) 679 else: 680 ind = 1 681 if zeros: 682 ind = 0 683 if self.exclusive: 684 count_dictionary = dict(Counter(self.data.flatten().cpu().numpy())) 685 else: 686 d = {} 687 for i in range(self.data.shape[1]): 688 cnt = Counter(self.data[:, i, :].flatten().cpu().numpy()) 689 d[i] = cnt[ind] 690 count_dictionary = d 691 if perc: 692 total = sum([v for k, v in count_dictionary.items()]) 693 count_dictionary = {k: v / total for k, v in count_dictionary.items()} 694 for i in self.behaviors_dict(): 695 if i not in count_dictionary: 696 count_dictionary[i] = 0 697 return count_dictionary
Get a dictionary with class-wise frame counts
Parameters
perc : bool, default False
if True
, a fraction of the total frame count is returned
zeros : bool, default False
if True
and annotation is not exclusive, zero counts are returned
bouts : bool, default False
if True
, instead of frame counts segment counts are returned
Returns
count_dictionary : dict a dictionary with class indices as keys and frame counts as values
699 def behaviors_dict(self) -> Dict: 700 """ 701 Get a dictionary of class names 702 703 Returns 704 ------- 705 behavior_dictionary: dict 706 a dictionary with class indices as keys and class names as values 707 """ 708 709 # if self.behaviors is None 710 if self.exclusive and "other" not in self.behaviors: 711 d = {i + 1: b for i, b in enumerate(self.behaviors)} 712 d[0] = "other" 713 else: 714 d = {i: b for i, b in enumerate(self.behaviors)} 715 return d
Get a dictionary of class names
Returns
behavior_dictionary: dict a dictionary with class indices as keys and class names as values
717 def annotation_class(self) -> str: 718 """ 719 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification') 720 721 Returns 722 ------- 723 annotation_class : str 724 the type of annotation 725 """ 726 727 if self.exclusive: 728 return "exclusive_classification" 729 else: 730 return "nonexclusive_classification"
Get the type of annotation ('exclusive_classification', 'nonexclusive_classification')
Returns
annotation_class : str the type of annotation
732 def size(self) -> int: 733 """ 734 Get the total number of frames in the data 735 736 Returns 737 ------- 738 size : int 739 the total number of frames 740 """ 741 742 return self.data.shape[0] * self.data.shape[-1]
Get the total number of frames in the data
Returns
size : int the total number of frames
744 def filtered_indices(self) -> List: 745 """ 746 Return the indices of the samples that should be removed 747 748 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 749 parameters 750 751 Returns 752 ------- 753 indices_to_remove : list 754 a list of integer indices that should be removed 755 """ 756 757 return self.filtered
Return the indices of the samples that should be removed
Choosing the indices can be based on any kind of filering defined in the __init__ function by the data parameters
Returns
indices_to_remove : list a list of integer indices that should be removed
759 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 760 """ 761 Set pseudo labels to the unlabeled data 762 763 Parameters 764 ---------- 765 labels : torch.Tensor 766 a tensor of pseudo-labels for the unlabeled data 767 """ 768 769 self.data[self.unlabeled_indices] = labels
Set pseudo labels to the unlabeled data
Parameters
labels : torch.Tensor a tensor of pseudo-labels for the unlabeled data
771 @classmethod 772 def get_file_ids( 773 cls, 774 annotation_path: Union[str, Set], 775 annotation_suffix: Union[str, Set], 776 *args, 777 **kwargs, 778 ) -> List: 779 """ 780 Process data parameters and return a list of ids of the videos that should 781 be processed by the __init__ function 782 783 Parameters 784 ---------- 785 annotation_path : str | set 786 the path or the set of paths to the folder where the annotation files are stored 787 annotation_suffix : str | set, optional 788 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 789 790 Returns 791 ------- 792 video_ids : list 793 a list of video file ids 794 """ 795 796 lists = [annotation_path, annotation_suffix] 797 for i in range(len(lists)): 798 iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str)) 799 if lists[i] is not None: 800 if not iterable: 801 lists[i] = [lists[i]] 802 lists[i] = [x for x in lists[i]] 803 annotation_path, annotation_suffix = lists 804 files = [] 805 for folder in annotation_path: 806 files += [ 807 strip_suffix(os.path.basename(file), annotation_suffix) 808 for file in os.listdir(folder) 809 if file.endswith(tuple([x for x in annotation_suffix])) 810 ] 811 files = sorted(files, key=lambda x: os.path.basename(x)) 812 return files
Process data parameters and return a list of ids of the videos that should be processed by the __init__ function
Parameters
annotation_path : str | set the path or the set of paths to the folder where the annotation files are stored annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
Returns
video_ids : list a list of video file ids
1097class FileAnnotationStore(ActionSegmentationStore): # + 1098 """ 1099 A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to one video 1100 """ 1101 1102 def _generate_max_min_frames(self, times: Dict, video_id: str) -> None: 1103 """ 1104 Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore` 1105 """ 1106 1107 if video_id in self.max_frames: 1108 return 1109 for ind, cat_dict in times.items(): 1110 maxes = [] 1111 mins = [] 1112 for cat, cat_list in cat_dict.items(): 1113 if len(cat_list) > 0: 1114 maxes.append(max([x[1] for x in cat_list])) 1115 mins.append(min([x[0] for x in cat_list])) 1116 self.max_frames[video_id][ind] = max(maxes) 1117 self.min_frames[video_id][ind] = min(mins) 1118 1119 def _load_data(self) -> torch.Tensor: 1120 """ 1121 Load behavior annotation and generate annotation prompts 1122 """ 1123 1124 if self.video_order is None: 1125 return None 1126 1127 files = [] 1128 for x in self.video_order: 1129 ok = False 1130 for folder in self.annotation_folder: 1131 for s in self.ann_suffix: 1132 file = os.path.join(folder, x + s) 1133 if os.path.exists(file): 1134 files.append(file) 1135 ok = True 1136 break 1137 if not ok: 1138 files.append(None) 1139 self.not_found = set() 1140 annotations_dict = {} 1141 print("Computing annotation arrays...") 1142 for name, filename in tqdm(list(zip(self.video_order, files))): 1143 if filename is not None: 1144 times = self._open_annotations(filename) 1145 else: 1146 times = None 1147 if times is not None: 1148 self._generate_max_min_frames(times, name) 1149 annotations_dict.update(self._generate_annotation(times, name)) 1150 del times 1151 annotation = self._make_trimmed_annotations(annotations_dict) 1152 del annotations_dict 1153 if self.filter_annotated: 1154 if self.exclusive: 1155 s = torch.sum((annotation != -100), dim=1) 1156 else: 1157 s = torch.sum( 1158 torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1 1159 ) 1160 self.filtered += torch.where(s == 0)[0].tolist() 1161 annotation[annotation == -200] = -100 1162 return annotation 1163 1164 @abstractmethod 1165 def _open_annotations(self, filename: str) -> Dict: 1166 """ 1167 Load the annotation from filename 1168 1169 Parameters 1170 ---------- 1171 filename : str 1172 path to an annotation file 1173 1174 Returns 1175 ------- 1176 times : dict 1177 a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are 1178 lists of (start, end, ambiguity status) lists 1179 """
A generalized implementation of ActionSegmentationStore
for datasets where one file corresponds to one video
Inherited Members
1182class SequenceAnnotationStore(ActionSegmentationStore): # + 1183 """ 1184 A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to multiple videos 1185 """ 1186 1187 def _generate_max_min_frames(self, times: Dict) -> None: 1188 """ 1189 Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore` 1190 """ 1191 1192 for video_id in times: 1193 if video_id in self.max_frames: 1194 continue 1195 self.max_frames[video_id] = {} 1196 for ind, cat_dict in times[video_id].items(): 1197 maxes = [] 1198 mins = [] 1199 for cat, cat_list in cat_dict.items(): 1200 maxes.append(max([x[1] for x in cat_list])) 1201 mins.append(min([x[0] for x in cat_list])) 1202 self.max_frames[video_id][ind] = max(maxes) 1203 self.min_frames[video_id][ind] = min(mins) 1204 1205 @classmethod 1206 def get_file_ids( 1207 cls, 1208 filenames: List = None, 1209 annotation_path: str = None, 1210 *args, 1211 **kwargs, 1212 ) -> List: 1213 """ 1214 Process data parameters and return a list of ids of the videos that should 1215 be processed by the __init__ function 1216 1217 Parameters 1218 ---------- 1219 filenames : list, optional 1220 a list of annotation file paths 1221 annotation_path : str, optional 1222 path to the annotation folder 1223 1224 Returns 1225 ------- 1226 video_ids : list 1227 a list of video file ids 1228 """ 1229 1230 file_paths = [] 1231 if annotation_path is not None: 1232 if isinstance(annotation_path, str): 1233 annotation_path = [annotation_path] 1234 file_paths = [] 1235 for folder in annotation_path: 1236 file_paths += [os.path.join(folder, x) for x in os.listdir(folder)] 1237 ids = set() 1238 for f in file_paths: 1239 if os.path.basename(f) in filenames: 1240 ids.add(os.path.basename(f)) 1241 ids = sorted(ids) 1242 return ids 1243 1244 def _load_data(self) -> torch.Tensor: 1245 """ 1246 Load behavior annotation and generate annotation prompts 1247 """ 1248 1249 if self.video_order is None: 1250 return None 1251 1252 files = [] 1253 for f in self.file_paths: 1254 if os.path.basename(f) in self.video_order: 1255 files.append(f) 1256 self.not_found = set() 1257 annotations_dict = {} 1258 for name, filename in tqdm(zip(self.video_order, files)): 1259 if filename is not None: 1260 times = self._open_sequences(filename) 1261 else: 1262 times = None 1263 if times is not None: 1264 self._generate_max_min_frames(times) 1265 none_ids = [] 1266 for video_id, sequence_dict in times.items(): 1267 if sequence_dict is None: 1268 none_ids.append(sequence_dict) 1269 continue 1270 annotations_dict.update( 1271 self._generate_annotation(sequence_dict, video_id) 1272 ) 1273 for video_id in none_ids: 1274 annotations_dict.update(self._generate_annotation(None, video_id)) 1275 del times 1276 annotation = self._make_trimmed_annotations(annotations_dict) 1277 del annotations_dict 1278 if self.filter_annotated: 1279 if self.exclusive: 1280 s = torch.sum((annotation != -100), dim=1) 1281 else: 1282 s = torch.sum( 1283 torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1 1284 ) 1285 self.filtered += torch.where(s == 0)[0].tolist() 1286 annotation[annotation == -200] = -100 1287 return annotation 1288 1289 @abstractmethod 1290 def _open_sequences(self, filename: str) -> Dict: 1291 """ 1292 Load the annotation from filename 1293 1294 Parameters 1295 ---------- 1296 filename : str 1297 path to an annotation file 1298 1299 Returns 1300 ------- 1301 times : dict 1302 a nested dictionary where first-level keys are video ids, second-level keys are clip ids, 1303 third-level keys are categories and values are 1304 lists of (start, end, ambiguity status) lists 1305 """
A generalized implementation of ActionSegmentationStore
for datasets where one file corresponds to multiple videos
1205 @classmethod 1206 def get_file_ids( 1207 cls, 1208 filenames: List = None, 1209 annotation_path: str = None, 1210 *args, 1211 **kwargs, 1212 ) -> List: 1213 """ 1214 Process data parameters and return a list of ids of the videos that should 1215 be processed by the __init__ function 1216 1217 Parameters 1218 ---------- 1219 filenames : list, optional 1220 a list of annotation file paths 1221 annotation_path : str, optional 1222 path to the annotation folder 1223 1224 Returns 1225 ------- 1226 video_ids : list 1227 a list of video file ids 1228 """ 1229 1230 file_paths = [] 1231 if annotation_path is not None: 1232 if isinstance(annotation_path, str): 1233 annotation_path = [annotation_path] 1234 file_paths = [] 1235 for folder in annotation_path: 1236 file_paths += [os.path.join(folder, x) for x in os.listdir(folder)] 1237 ids = set() 1238 for f in file_paths: 1239 if os.path.basename(f) in filenames: 1240 ids.add(os.path.basename(f)) 1241 ids = sorted(ids) 1242 return ids
Process data parameters and return a list of ids of the videos that should be processed by the __init__ function
Parameters
filenames : list, optional a list of annotation file paths annotation_path : str, optional path to the annotation folder
Returns
video_ids : list a list of video file ids
Inherited Members
1308class DLCAnnotationStore(FileAnnotationStore): # + 1309 """ 1310 DLC type annotation data 1311 1312 The files are either the DLC2Action GUI output or a pickled dictionary of the following structure: 1313 - nested dictionary, 1314 - first-level keys are individual IDs, 1315 - second-level keys are labels, 1316 - values are lists of intervals, 1317 - the lists of intervals is formatted as `[start_frame, end_frame, ambiguity]`, 1318 - ambiguity is 1 if the action is ambiguous (!!at the moment DLC2Action will IGNORE those intervals!!) or 0 if it isn't. 1319 1320 A minimum working example of such a dictionary is: 1321 ``` 1322 { 1323 "ind0": {}, 1324 "ind1": { 1325 "running": [60, 70, 0]], 1326 "eating": [] 1327 } 1328 } 1329 ``` 1330 1331 Here there are two animals: `"ind0"` and `"ind1"`, and two actions: running and eating. 1332 The only annotated action is eating for `"ind1"` between frames 60 and 70. 1333 1334 If you generate those files manually, run this code for a sanity check: 1335 ``` 1336 import pickle 1337 1338 with open("/path/to/annotation.pickle", "rb") as f: 1339 data = pickle.load(f) 1340 1341 for ind, ind_dict in data.items(): 1342 print(f'individual {ind}:') 1343 for label, intervals in ind_dict.items(): 1344 for start, end, ambiguity in intervals: 1345 if ambiguity == 0: 1346 print(f' from {start} to {end} frame: {label}') 1347 ``` 1348 1349 Assumes the following file structure: 1350 ``` 1351 annotation_path 1352 ├── video1_annotation.pickle 1353 └── video2_labels.pickle 1354 ``` 1355 Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`. 1356 """ 1357 1358 def _open_annotations(self, filename: str) -> Dict: 1359 """ 1360 Load the annotation from `filename` 1361 """ 1362 1363 try: 1364 with open(filename, "rb") as f: 1365 data = pickle.load(f) 1366 if isinstance(data, dict): 1367 annotation = data 1368 for ind in annotation: 1369 for cat, cat_list in annotation[ind].items(): 1370 annotation[ind][cat] = [ 1371 [start, end, 0] for start, end in cat_list 1372 ] 1373 else: 1374 _, loaded_labels, animals, loaded_times = data 1375 annotation = {} 1376 for ind, ind_list in zip(animals, loaded_times): 1377 annotation[ind] = {} 1378 for cat, cat_list in zip(loaded_labels, ind_list): 1379 annotation[ind][cat] = cat_list 1380 return annotation 1381 except: 1382 print(f"{filename} is invalid or does not exist") 1383 return None
DLC type annotation data
The files are either the DLC2Action GUI output or a pickled dictionary of the following structure:
- nested dictionary,
- first-level keys are individual IDs,
- second-level keys are labels,
- values are lists of intervals,
- the lists of intervals is formatted as [start_frame, end_frame, ambiguity]
,
- ambiguity is 1 if the action is ambiguous (!!at the moment DLC2Action will IGNORE those intervals!!) or 0 if it isn't.
A minimum working example of such a dictionary is:
{
"ind0": {},
"ind1": {
"running": [60, 70, 0]],
"eating": []
}
}
Here there are two animals: "ind0"
and "ind1"
, and two actions: running and eating.
The only annotated action is eating for "ind1"
between frames 60 and 70.
If you generate those files manually, run this code for a sanity check:
import pickle
with open("/path/to/annotation.pickle", "rb") as f:
data = pickle.load(f)
for ind, ind_dict in data.items():
print(f'individual {ind}:')
for label, intervals in ind_dict.items():
for start, end, ambiguity in intervals:
if ambiguity == 0:
print(f' from {start} to {end} frame: {label}')
Assumes the following file structure:
annotation_path
├── video1_annotation.pickle
└── video2_labels.pickle
Here annotation_suffix
is {'_annotation.pickle', '_labels.pickle'}
.
Inherited Members
1386class BorisAnnotationStore(FileAnnotationStore): # + 1387 """ 1388 BORIS type annotation data 1389 1390 Assumes the following file structure: 1391 ``` 1392 annotation_path 1393 ├── video1_annotation.pickle 1394 └── video2_labels.pickle 1395 ``` 1396 Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`. 1397 """ 1398 1399 def __init__( 1400 self, 1401 video_order: List = None, 1402 min_frames: Dict = None, 1403 max_frames: Dict = None, 1404 visibility: Dict = None, 1405 exclusive: bool = True, 1406 len_segment: int = 128, 1407 overlap: int = 0, 1408 behaviors: Set = None, 1409 ignored_classes: Set = None, 1410 annotation_suffix: Union[Set, str] = None, 1411 annotation_path: Union[Set, str] = None, 1412 behavior_file: str = None, 1413 correction: Dict = None, 1414 frame_limit: int = 0, 1415 filter_annotated: bool = False, 1416 filter_background: bool = False, 1417 error_class: str = None, 1418 min_frames_action: int = None, 1419 key_objects: Tuple = None, 1420 visibility_min_score: float = 0.2, 1421 visibility_min_frac: float = 0.7, 1422 mask: Dict = None, 1423 use_hard_negatives: bool = False, 1424 default_agent_name: str = "ind0", 1425 interactive: bool = False, 1426 ignored_clips: Set = None, 1427 *args, 1428 **kwargs, 1429 ) -> None: 1430 """ 1431 Parameters 1432 ---------- 1433 video_order : list, optional 1434 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1435 min_frames : dict, optional 1436 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1437 clip start frames (not passed if creating from key objects) 1438 max_frames : dict, optional 1439 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1440 clip end frames (not passed if creating from key objects) 1441 visibility : dict, optional 1442 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1443 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 1444 exclusive : bool, default True 1445 if True, the annotation is single-label; if False, multi-label 1446 len_segment : int, default 128 1447 the length of the segments in which the data should be cut (in frames) 1448 overlap : int, default 0 1449 the length of the overlap between neighboring segments (in frames) 1450 behaviors : set, optional 1451 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 1452 loaded from a file) 1453 ignored_classes : set, optional 1454 the list of behaviors from the behaviors list or file to not annotate 1455 annotation_suffix : str | set, optional 1456 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 1457 (not passed if creating from key objects or if irrelevant for the dataset) 1458 annotation_path : str | set, optional 1459 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1460 from key objects) 1461 behavior_file : str, optional 1462 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 1463 correction : dict, optional 1464 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 1465 can be used to correct for variations in naming or to merge several labels in one 1466 frame_limit : int, default 0 1467 the smallest possible length of a clip (shorter clips are discarded) 1468 filter_annotated : bool, default False 1469 if True, the samples that do not have any labels will be filtered 1470 filter_background : bool, default False 1471 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 1472 error_class : str, optional 1473 the name of the error class (the annotations that intersect with this label will be discarded) 1474 min_frames_action : int, default 0 1475 the minimum length of an action (shorter actions are not annotated) 1476 key_objects : tuple, optional 1477 the key objects to load the AnnotationStore from 1478 visibility_min_score : float, default 5 1479 the minimum visibility score for visibility filtering 1480 visibility_min_frac : float, default 0.7 1481 the minimum fraction of visible frames for visibility filtering 1482 mask : dict, optional 1483 a masked value dictionary (for active learning simulation experiments) 1484 use_hard_negatives : bool, default False 1485 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 1486 interactive : bool, default False 1487 if `True`, annotation is assigned to pairs of individuals 1488 ignored_clips : set, optional 1489 a set of clip ids to ignore 1490 """ 1491 1492 self.default_agent_name = default_agent_name 1493 super().__init__( 1494 video_order=video_order, 1495 min_frames=min_frames, 1496 max_frames=max_frames, 1497 visibility=visibility, 1498 exclusive=exclusive, 1499 len_segment=len_segment, 1500 overlap=overlap, 1501 behaviors=behaviors, 1502 ignored_classes=ignored_classes, 1503 annotation_suffix=annotation_suffix, 1504 annotation_path=annotation_path, 1505 behavior_file=behavior_file, 1506 correction=correction, 1507 frame_limit=frame_limit, 1508 filter_annotated=filter_annotated, 1509 filter_background=filter_background, 1510 error_class=error_class, 1511 min_frames_action=min_frames_action, 1512 key_objects=key_objects, 1513 visibility_min_score=visibility_min_score, 1514 visibility_min_frac=visibility_min_frac, 1515 mask=mask, 1516 use_hard_negatives=use_hard_negatives, 1517 interactive=interactive, 1518 ignored_clips=ignored_clips, 1519 ) 1520 1521 def _open_annotations(self, filename: str) -> Dict: 1522 """ 1523 Load the annotation from filename 1524 """ 1525 1526 try: 1527 df = pd.read_csv(filename, header=15) 1528 fps = df.iloc[0]["FPS"] 1529 df["Subject"] = df["Subject"].fillna(self.default_agent_name) 1530 loaded_labels = list(df["Behavior"].unique()) 1531 animals = list(df["Subject"].unique()) 1532 loaded_times = {} 1533 for ind in animals: 1534 loaded_times[ind] = {} 1535 agent_df = df[df["Subject"] == ind] 1536 for cat in loaded_labels: 1537 filtered_df = agent_df[agent_df["Behavior"] == cat] 1538 starts = ( 1539 filtered_df["Time"][filtered_df["Status"] == "START"] * fps 1540 ).astype(int) 1541 ends = ( 1542 filtered_df["Time"][filtered_df["Status"] == "STOP"] * fps 1543 ).astype(int) 1544 loaded_times[ind][cat] = [ 1545 [start, end, 0] for start, end in zip(starts, ends) 1546 ] 1547 return loaded_times 1548 except: 1549 print(f"{filename} is invalid or does not exist") 1550 return None
BORIS type annotation data
Assumes the following file structure:
annotation_path
├── video1_annotation.pickle
└── video2_labels.pickle
Here annotation_suffix
is {'_annotation.pickle', '_labels.pickle'}
.
1399 def __init__( 1400 self, 1401 video_order: List = None, 1402 min_frames: Dict = None, 1403 max_frames: Dict = None, 1404 visibility: Dict = None, 1405 exclusive: bool = True, 1406 len_segment: int = 128, 1407 overlap: int = 0, 1408 behaviors: Set = None, 1409 ignored_classes: Set = None, 1410 annotation_suffix: Union[Set, str] = None, 1411 annotation_path: Union[Set, str] = None, 1412 behavior_file: str = None, 1413 correction: Dict = None, 1414 frame_limit: int = 0, 1415 filter_annotated: bool = False, 1416 filter_background: bool = False, 1417 error_class: str = None, 1418 min_frames_action: int = None, 1419 key_objects: Tuple = None, 1420 visibility_min_score: float = 0.2, 1421 visibility_min_frac: float = 0.7, 1422 mask: Dict = None, 1423 use_hard_negatives: bool = False, 1424 default_agent_name: str = "ind0", 1425 interactive: bool = False, 1426 ignored_clips: Set = None, 1427 *args, 1428 **kwargs, 1429 ) -> None: 1430 """ 1431 Parameters 1432 ---------- 1433 video_order : list, optional 1434 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1435 min_frames : dict, optional 1436 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1437 clip start frames (not passed if creating from key objects) 1438 max_frames : dict, optional 1439 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1440 clip end frames (not passed if creating from key objects) 1441 visibility : dict, optional 1442 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1443 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 1444 exclusive : bool, default True 1445 if True, the annotation is single-label; if False, multi-label 1446 len_segment : int, default 128 1447 the length of the segments in which the data should be cut (in frames) 1448 overlap : int, default 0 1449 the length of the overlap between neighboring segments (in frames) 1450 behaviors : set, optional 1451 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 1452 loaded from a file) 1453 ignored_classes : set, optional 1454 the list of behaviors from the behaviors list or file to not annotate 1455 annotation_suffix : str | set, optional 1456 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 1457 (not passed if creating from key objects or if irrelevant for the dataset) 1458 annotation_path : str | set, optional 1459 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1460 from key objects) 1461 behavior_file : str, optional 1462 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 1463 correction : dict, optional 1464 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 1465 can be used to correct for variations in naming or to merge several labels in one 1466 frame_limit : int, default 0 1467 the smallest possible length of a clip (shorter clips are discarded) 1468 filter_annotated : bool, default False 1469 if True, the samples that do not have any labels will be filtered 1470 filter_background : bool, default False 1471 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 1472 error_class : str, optional 1473 the name of the error class (the annotations that intersect with this label will be discarded) 1474 min_frames_action : int, default 0 1475 the minimum length of an action (shorter actions are not annotated) 1476 key_objects : tuple, optional 1477 the key objects to load the AnnotationStore from 1478 visibility_min_score : float, default 5 1479 the minimum visibility score for visibility filtering 1480 visibility_min_frac : float, default 0.7 1481 the minimum fraction of visible frames for visibility filtering 1482 mask : dict, optional 1483 a masked value dictionary (for active learning simulation experiments) 1484 use_hard_negatives : bool, default False 1485 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 1486 interactive : bool, default False 1487 if `True`, annotation is assigned to pairs of individuals 1488 ignored_clips : set, optional 1489 a set of clip ids to ignore 1490 """ 1491 1492 self.default_agent_name = default_agent_name 1493 super().__init__( 1494 video_order=video_order, 1495 min_frames=min_frames, 1496 max_frames=max_frames, 1497 visibility=visibility, 1498 exclusive=exclusive, 1499 len_segment=len_segment, 1500 overlap=overlap, 1501 behaviors=behaviors, 1502 ignored_classes=ignored_classes, 1503 annotation_suffix=annotation_suffix, 1504 annotation_path=annotation_path, 1505 behavior_file=behavior_file, 1506 correction=correction, 1507 frame_limit=frame_limit, 1508 filter_annotated=filter_annotated, 1509 filter_background=filter_background, 1510 error_class=error_class, 1511 min_frames_action=min_frames_action, 1512 key_objects=key_objects, 1513 visibility_min_score=visibility_min_score, 1514 visibility_min_frac=visibility_min_frac, 1515 mask=mask, 1516 use_hard_negatives=use_hard_negatives, 1517 interactive=interactive, 1518 ignored_clips=ignored_clips, 1519 )
Parameters
video_order : list, optional
a list of video ids that should be processed in the same order (not passed if creating from key objects)
min_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip start frames (not passed if creating from key objects)
max_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip end frames (not passed if creating from key objects)
visibility : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
exclusive : bool, default True
if True, the annotation is single-label; if False, multi-label
len_segment : int, default 128
the length of the segments in which the data should be cut (in frames)
overlap : int, default 0
the length of the overlap between neighboring segments (in frames)
behaviors : set, optional
the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
loaded from a file)
ignored_classes : set, optional
the list of behaviors from the behaviors list or file to not annotate
annotation_suffix : str | set, optional
the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
(not passed if creating from key objects or if irrelevant for the dataset)
annotation_path : str | set, optional
the path or the set of paths to the folder where the annotation files are stored (not passed if creating
from key objects)
behavior_file : str, optional
the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
correction : dict, optional
a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
can be used to correct for variations in naming or to merge several labels in one
frame_limit : int, default 0
the smallest possible length of a clip (shorter clips are discarded)
filter_annotated : bool, default False
if True, the samples that do not have any labels will be filtered
filter_background : bool, default False
if True, only the unlabeled frames that are close to annotated frames will be labeled as background
error_class : str, optional
the name of the error class (the annotations that intersect with this label will be discarded)
min_frames_action : int, default 0
the minimum length of an action (shorter actions are not annotated)
key_objects : tuple, optional
the key objects to load the AnnotationStore from
visibility_min_score : float, default 5
the minimum visibility score for visibility filtering
visibility_min_frac : float, default 0.7
the minimum fraction of visible frames for visibility filtering
mask : dict, optional
a masked value dictionary (for active learning simulation experiments)
use_hard_negatives : bool, default False
mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
interactive : bool, default False
if True
, annotation is assigned to pairs of individuals
ignored_clips : set, optional
a set of clip ids to ignore
Inherited Members
1553class PKUMMDAnnotationStore(FileAnnotationStore): # + 1554 """ 1555 PKU-MMD annotation data 1556 1557 Assumes the following file structure: 1558 ``` 1559 annotation_path 1560 ├── 0364-L.txt 1561 ... 1562 └── 0144-M.txt 1563 ``` 1564 """ 1565 1566 def __init__( 1567 self, 1568 video_order: List = None, 1569 min_frames: Dict = None, 1570 max_frames: Dict = None, 1571 visibility: Dict = None, 1572 exclusive: bool = True, 1573 len_segment: int = 128, 1574 overlap: int = 0, 1575 ignored_classes: Set = None, 1576 annotation_path: Union[Set, str] = None, 1577 behavior_file: str = None, 1578 correction: Dict = None, 1579 frame_limit: int = 0, 1580 filter_annotated: bool = False, 1581 filter_background: bool = False, 1582 error_class: str = None, 1583 min_frames_action: int = None, 1584 key_objects: Tuple = None, 1585 visibility_min_score: float = 0, 1586 visibility_min_frac: float = 0, 1587 mask: Dict = None, 1588 use_hard_negatives: bool = False, 1589 interactive: bool = False, 1590 ignored_clips: Set = None, 1591 *args, 1592 **kwargs, 1593 ) -> None: 1594 """ 1595 Parameters 1596 ---------- 1597 video_order : list, optional 1598 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1599 min_frames : dict, optional 1600 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1601 clip start frames (not passed if creating from key objects) 1602 max_frames : dict, optional 1603 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1604 clip end frames (not passed if creating from key objects) 1605 visibility : dict, optional 1606 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1607 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 1608 exclusive : bool, default True 1609 if True, the annotation is single-label; if False, multi-label 1610 len_segment : int, default 128 1611 the length of the segments in which the data should be cut (in frames) 1612 overlap : int, default 0 1613 the length of the overlap between neighboring segments (in frames) 1614 ignored_classes : set, optional 1615 the list of behaviors from the behaviors list or file to not annotate 1616 annotation_path : str | set, optional 1617 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1618 from key objects) 1619 behavior_file : str, optional 1620 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 1621 correction : dict, optional 1622 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 1623 can be used to correct for variations in naming or to merge several labels in one 1624 frame_limit : int, default 0 1625 the smallest possible length of a clip (shorter clips are discarded) 1626 filter_annotated : bool, default False 1627 if True, the samples that do not have any labels will be filtered 1628 filter_background : bool, default False 1629 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 1630 error_class : str, optional 1631 the name of the error class (the annotations that intersect with this label will be discarded) 1632 min_frames_action : int, default 0 1633 the minimum length of an action (shorter actions are not annotated) 1634 key_objects : tuple, optional 1635 the key objects to load the AnnotationStore from 1636 visibility_min_score : float, default 5 1637 the minimum visibility score for visibility filtering 1638 visibility_min_frac : float, default 0.7 1639 the minimum fraction of visible frames for visibility filtering 1640 mask : dict, optional 1641 a masked value dictionary (for active learning simulation experiments) 1642 use_hard_negatives : bool, default False 1643 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 1644 interactive : bool, default False 1645 if `True`, annotation is assigned to pairs of individuals 1646 ignored_clips : set, optional 1647 a set of clip ids to ignore 1648 """ 1649 1650 super().__init__( 1651 video_order=video_order, 1652 min_frames=min_frames, 1653 max_frames=max_frames, 1654 visibility=visibility, 1655 exclusive=exclusive, 1656 len_segment=len_segment, 1657 overlap=overlap, 1658 behaviors=None, 1659 ignored_classes=ignored_classes, 1660 annotation_suffix={".txt"}, 1661 annotation_path=annotation_path, 1662 behavior_file=behavior_file, 1663 correction=correction, 1664 frame_limit=frame_limit, 1665 filter_annotated=filter_annotated, 1666 filter_background=filter_background, 1667 error_class=error_class, 1668 min_frames_action=min_frames_action, 1669 key_objects=key_objects, 1670 visibility_min_score=visibility_min_score, 1671 visibility_min_frac=visibility_min_frac, 1672 mask=mask, 1673 use_hard_negatives=use_hard_negatives, 1674 interactive=interactive, 1675 ignored_clips=ignored_clips, 1676 *args, 1677 **kwargs, 1678 ) 1679 1680 @classmethod 1681 def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List: 1682 """ 1683 Process data parameters and return a list of ids of the videos that should 1684 be processed by the __init__ function 1685 1686 Parameters 1687 ---------- 1688 annotation_path : str | set 1689 the path or the set of paths to the folder where the annotation files are stored 1690 1691 Returns 1692 ------- 1693 video_ids : list 1694 a list of video file ids 1695 """ 1696 1697 if isinstance(annotation_path, str): 1698 annotation_path = [annotation_path] 1699 files = [] 1700 for folder in annotation_path: 1701 files += [ 1702 os.path.basename(x)[:-4] for x in os.listdir(folder) if x[-4:] == ".txt" 1703 ] 1704 files = sorted(files, key=lambda x: os.path.basename(x)) 1705 return files 1706 1707 def _open_annotations(self, filename: str) -> Dict: 1708 """ 1709 Load the annotation from filename 1710 """ 1711 1712 if self.interactive: 1713 agent_name = "0+1" 1714 else: 1715 agent_name = "0" 1716 times = {agent_name: defaultdict(lambda: [])} 1717 with open(filename) as f: 1718 for line in f.readlines(): 1719 label, start, end, *_ = map(int, line.split(",")) 1720 times[agent_name][self.all_behaviors[int(label) - 1]].append( 1721 [start, end, 0] 1722 ) 1723 return times 1724 1725 def _set_behaviors( 1726 self, behaviors: List, ignored_classes: List, behavior_file: str 1727 ): 1728 """ 1729 Get a list of behaviors that should be annotated from behavior parameters 1730 """ 1731 1732 if behavior_file is not None: 1733 behaviors = list(pd.read_excel(behavior_file)["Action"]) 1734 self.all_behaviors = copy(behaviors) 1735 for b in ignored_classes: 1736 if b in behaviors: 1737 behaviors.remove(b) 1738 self.behaviors = behaviors
PKU-MMD annotation data
Assumes the following file structure:
annotation_path
├── 0364-L.txt
...
└── 0144-M.txt
1566 def __init__( 1567 self, 1568 video_order: List = None, 1569 min_frames: Dict = None, 1570 max_frames: Dict = None, 1571 visibility: Dict = None, 1572 exclusive: bool = True, 1573 len_segment: int = 128, 1574 overlap: int = 0, 1575 ignored_classes: Set = None, 1576 annotation_path: Union[Set, str] = None, 1577 behavior_file: str = None, 1578 correction: Dict = None, 1579 frame_limit: int = 0, 1580 filter_annotated: bool = False, 1581 filter_background: bool = False, 1582 error_class: str = None, 1583 min_frames_action: int = None, 1584 key_objects: Tuple = None, 1585 visibility_min_score: float = 0, 1586 visibility_min_frac: float = 0, 1587 mask: Dict = None, 1588 use_hard_negatives: bool = False, 1589 interactive: bool = False, 1590 ignored_clips: Set = None, 1591 *args, 1592 **kwargs, 1593 ) -> None: 1594 """ 1595 Parameters 1596 ---------- 1597 video_order : list, optional 1598 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1599 min_frames : dict, optional 1600 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1601 clip start frames (not passed if creating from key objects) 1602 max_frames : dict, optional 1603 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1604 clip end frames (not passed if creating from key objects) 1605 visibility : dict, optional 1606 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1607 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 1608 exclusive : bool, default True 1609 if True, the annotation is single-label; if False, multi-label 1610 len_segment : int, default 128 1611 the length of the segments in which the data should be cut (in frames) 1612 overlap : int, default 0 1613 the length of the overlap between neighboring segments (in frames) 1614 ignored_classes : set, optional 1615 the list of behaviors from the behaviors list or file to not annotate 1616 annotation_path : str | set, optional 1617 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1618 from key objects) 1619 behavior_file : str, optional 1620 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 1621 correction : dict, optional 1622 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 1623 can be used to correct for variations in naming or to merge several labels in one 1624 frame_limit : int, default 0 1625 the smallest possible length of a clip (shorter clips are discarded) 1626 filter_annotated : bool, default False 1627 if True, the samples that do not have any labels will be filtered 1628 filter_background : bool, default False 1629 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 1630 error_class : str, optional 1631 the name of the error class (the annotations that intersect with this label will be discarded) 1632 min_frames_action : int, default 0 1633 the minimum length of an action (shorter actions are not annotated) 1634 key_objects : tuple, optional 1635 the key objects to load the AnnotationStore from 1636 visibility_min_score : float, default 5 1637 the minimum visibility score for visibility filtering 1638 visibility_min_frac : float, default 0.7 1639 the minimum fraction of visible frames for visibility filtering 1640 mask : dict, optional 1641 a masked value dictionary (for active learning simulation experiments) 1642 use_hard_negatives : bool, default False 1643 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 1644 interactive : bool, default False 1645 if `True`, annotation is assigned to pairs of individuals 1646 ignored_clips : set, optional 1647 a set of clip ids to ignore 1648 """ 1649 1650 super().__init__( 1651 video_order=video_order, 1652 min_frames=min_frames, 1653 max_frames=max_frames, 1654 visibility=visibility, 1655 exclusive=exclusive, 1656 len_segment=len_segment, 1657 overlap=overlap, 1658 behaviors=None, 1659 ignored_classes=ignored_classes, 1660 annotation_suffix={".txt"}, 1661 annotation_path=annotation_path, 1662 behavior_file=behavior_file, 1663 correction=correction, 1664 frame_limit=frame_limit, 1665 filter_annotated=filter_annotated, 1666 filter_background=filter_background, 1667 error_class=error_class, 1668 min_frames_action=min_frames_action, 1669 key_objects=key_objects, 1670 visibility_min_score=visibility_min_score, 1671 visibility_min_frac=visibility_min_frac, 1672 mask=mask, 1673 use_hard_negatives=use_hard_negatives, 1674 interactive=interactive, 1675 ignored_clips=ignored_clips, 1676 *args, 1677 **kwargs, 1678 )
Parameters
video_order : list, optional
a list of video ids that should be processed in the same order (not passed if creating from key objects)
min_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip start frames (not passed if creating from key objects)
max_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip end frames (not passed if creating from key objects)
visibility : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
exclusive : bool, default True
if True, the annotation is single-label; if False, multi-label
len_segment : int, default 128
the length of the segments in which the data should be cut (in frames)
overlap : int, default 0
the length of the overlap between neighboring segments (in frames)
ignored_classes : set, optional
the list of behaviors from the behaviors list or file to not annotate
annotation_path : str | set, optional
the path or the set of paths to the folder where the annotation files are stored (not passed if creating
from key objects)
behavior_file : str, optional
the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
correction : dict, optional
a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
can be used to correct for variations in naming or to merge several labels in one
frame_limit : int, default 0
the smallest possible length of a clip (shorter clips are discarded)
filter_annotated : bool, default False
if True, the samples that do not have any labels will be filtered
filter_background : bool, default False
if True, only the unlabeled frames that are close to annotated frames will be labeled as background
error_class : str, optional
the name of the error class (the annotations that intersect with this label will be discarded)
min_frames_action : int, default 0
the minimum length of an action (shorter actions are not annotated)
key_objects : tuple, optional
the key objects to load the AnnotationStore from
visibility_min_score : float, default 5
the minimum visibility score for visibility filtering
visibility_min_frac : float, default 0.7
the minimum fraction of visible frames for visibility filtering
mask : dict, optional
a masked value dictionary (for active learning simulation experiments)
use_hard_negatives : bool, default False
mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
interactive : bool, default False
if True
, annotation is assigned to pairs of individuals
ignored_clips : set, optional
a set of clip ids to ignore
1680 @classmethod 1681 def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List: 1682 """ 1683 Process data parameters and return a list of ids of the videos that should 1684 be processed by the __init__ function 1685 1686 Parameters 1687 ---------- 1688 annotation_path : str | set 1689 the path or the set of paths to the folder where the annotation files are stored 1690 1691 Returns 1692 ------- 1693 video_ids : list 1694 a list of video file ids 1695 """ 1696 1697 if isinstance(annotation_path, str): 1698 annotation_path = [annotation_path] 1699 files = [] 1700 for folder in annotation_path: 1701 files += [ 1702 os.path.basename(x)[:-4] for x in os.listdir(folder) if x[-4:] == ".txt" 1703 ] 1704 files = sorted(files, key=lambda x: os.path.basename(x)) 1705 return files
Process data parameters and return a list of ids of the videos that should be processed by the __init__ function
Parameters
annotation_path : str | set the path or the set of paths to the folder where the annotation files are stored
Returns
video_ids : list a list of video file ids
Inherited Members
1741class CalMS21AnnotationStore(SequenceAnnotationStore): # + 1742 """ 1743 CalMS21 annotation data 1744 1745 Use the `'random:test_from_name:{name}'` and `'val-from-name:{val_name}:test-from-name:{test_name}'` 1746 partitioning methods with `'train'`, `'test'` and `'unlabeled'` names to separate into train, test and validation 1747 subsets according to the original files. For example, with `'val-from-name:test:test-from-name:unlabeled'` 1748 the data from the test file will go into validation and the unlabeled files will be the test. 1749 1750 Assumes the following file structure: 1751 ``` 1752 annotation_path 1753 ├── calms21_task_train.npy 1754 ├── calms21_task_test.npy 1755 ├── calms21_unlabeled_videos_part1.npy 1756 ├── calms21_unlabeled_videos_part2.npy 1757 └── calms21_unlabeled_videos_part3.npy 1758 ``` 1759 """ 1760 1761 def __init__( 1762 self, 1763 task_n: int = 1, 1764 include_task1: bool = True, 1765 video_order: List = None, 1766 min_frames: Dict = None, 1767 max_frames: Dict = None, 1768 len_segment: int = 128, 1769 overlap: int = 0, 1770 ignored_classes: Set = None, 1771 annotation_path: Union[Set, str] = None, 1772 key_objects: Tuple = None, 1773 treba_files: bool = False, 1774 *args, 1775 **kwargs, 1776 ) -> None: 1777 """ 1778 1779 Parameters 1780 ---------- 1781 task_n : [1, 2] 1782 the number of the task 1783 include_task1 : bool, default True 1784 include task 1 data to training set 1785 video_order : list, optional 1786 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1787 min_frames : dict, optional 1788 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1789 clip start frames (not passed if creating from key objects) 1790 max_frames : dict, optional 1791 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1792 clip end frames (not passed if creating from key objects) 1793 len_segment : int, default 128 1794 the length of the segments in which the data should be cut (in frames) 1795 overlap : int, default 0 1796 the length of the overlap between neighboring segments (in frames) 1797 ignored_classes : set, optional 1798 the list of behaviors from the behaviors list or file to not annotate 1799 annotation_path : str | set, optional 1800 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1801 from key objects) 1802 key_objects : tuple, optional 1803 the key objects to load the AnnotationStore from 1804 treba_files : bool, default False 1805 if `True`, TREBA feature files will be loaded 1806 """ 1807 1808 self.task_n = int(task_n) 1809 self.include_task1 = include_task1 1810 if self.task_n == 1: 1811 self.include_task1 = False 1812 self.treba_files = treba_files 1813 if "exclusive" in kwargs: 1814 exclusive = kwargs["exclusive"] 1815 else: 1816 exclusive = True 1817 if "behaviors" in kwargs and kwargs["behaviors"] is not None: 1818 behaviors = kwargs["behaviors"] 1819 else: 1820 behaviors = ["attack", "investigation", "mount", "other"] 1821 if task_n == 3: 1822 exclusive = False 1823 behaviors += [ 1824 "approach", 1825 "disengaged", 1826 "groom", 1827 "intromission", 1828 "mount_attempt", 1829 "sniff_face", 1830 "whiterearing", 1831 ] 1832 super().__init__( 1833 video_order=video_order, 1834 min_frames=min_frames, 1835 max_frames=max_frames, 1836 exclusive=exclusive, 1837 len_segment=len_segment, 1838 overlap=overlap, 1839 behaviors=behaviors, 1840 ignored_classes=ignored_classes, 1841 annotation_path=annotation_path, 1842 key_objects=key_objects, 1843 filter_annotated=False, 1844 interactive=True, 1845 ) 1846 1847 @classmethod 1848 def get_file_ids( 1849 cls, 1850 task_n: int = 1, 1851 include_task1: bool = False, 1852 treba_files: bool = False, 1853 annotation_path: Union[str, Set] = None, 1854 file_paths=None, 1855 *args, 1856 **kwargs, 1857 ) -> Iterable: 1858 """ 1859 Process data parameters and return a list of ids of the videos that should 1860 be processed by the __init__ function 1861 1862 Parameters 1863 ---------- 1864 task_n : {1, 2, 3} 1865 the index of the CalMS21 challenge task 1866 include_task1 : bool, default False 1867 if `True`, the training file of the task 1 will be loaded 1868 treba_files : bool, default False 1869 if `True`, the TREBA feature files will be loaded 1870 filenames : set, optional 1871 a set of string filenames to search for (only basenames, not the whole paths) 1872 annotation_path : str | set, optional 1873 the path to the folder where the pose and feature files are stored or a set of such paths 1874 (not passed if creating from key objects or from `file_paths`) 1875 file_paths : set, optional 1876 a set of string paths to the pose and feature files 1877 (not passed if creating from key objects or from `data_path`) 1878 1879 Returns 1880 ------- 1881 video_ids : list 1882 a list of video file ids 1883 """ 1884 1885 task_n = int(task_n) 1886 if task_n == 1: 1887 include_task1 = False 1888 files = [] 1889 if treba_files: 1890 postfix = "_features" 1891 else: 1892 postfix = "" 1893 files.append(f"calms21_task{task_n}_train{postfix}.npy") 1894 files.append(f"calms21_task{task_n}_test{postfix}.npy") 1895 if include_task1: 1896 files.append(f"calms21_task1_train{postfix}.npy") 1897 filenames = set(files) 1898 return SequenceAnnotationStore.get_file_ids( 1899 filenames, annotation_path=annotation_path 1900 ) 1901 1902 def _open_sequences(self, filename: str) -> Dict: 1903 """ 1904 Load the annotation from filename 1905 1906 Parameters 1907 ---------- 1908 filename : str 1909 path to an annotation file 1910 1911 Returns 1912 ------- 1913 times : dict 1914 a nested dictionary where first-level keys are video ids, second-level keys are clip ids, 1915 third-level keys are categories and values are 1916 lists of (start, end, ambiguity status) lists 1917 """ 1918 1919 data_dict = np.load(filename, allow_pickle=True).item() 1920 data = {} 1921 result = {} 1922 keys = list(data_dict.keys()) 1923 if "test" in os.path.basename(filename): 1924 mode = "test" 1925 elif "unlabeled" in os.path.basename(filename): 1926 mode = "unlabeled" 1927 else: 1928 mode = "train" 1929 if "approach" in keys: 1930 for behavior in keys: 1931 for key in data_dict[behavior].keys(): 1932 ann = data_dict[behavior][key]["annotations"] 1933 result[f'{mode}--{key.split("/")[-1]}'] = { 1934 "mouse1+mouse2": defaultdict(lambda: []) 1935 } 1936 starts = np.where( 1937 np.diff(np.concatenate([np.array([0]), ann, np.array([0])])) 1938 == 1 1939 )[0] 1940 ends = np.where( 1941 np.diff(np.concatenate([np.array([0]), ann, np.array([0])])) 1942 == -1 1943 )[0] 1944 for start, end in zip(starts, ends): 1945 result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][ 1946 behavior 1947 ].append([start, end, 0]) 1948 for b in self.behaviors: 1949 result[f'{mode}--{key.split("/")[-1]}---mouse1+mouse2'][ 1950 "mouse1+mouse2" 1951 ][f"unknown {b}"].append([0, len(ann), 0]) 1952 for key in keys: 1953 data.update(data_dict[key]) 1954 data_dict.pop(key) 1955 if "approach" not in keys and self.task_n == 3: 1956 for key in data.keys(): 1957 result[f'{mode}--{key.split("/")[-1]}'] = {"mouse1+mouse2": {}} 1958 ann = data[key]["annotations"] 1959 for i in range(4): 1960 starts = np.where( 1961 np.diff( 1962 np.concatenate( 1963 [np.array([0]), (ann == i).astype(int), np.array([0])] 1964 ) 1965 ) 1966 == 1 1967 )[0] 1968 ends = np.where( 1969 np.diff( 1970 np.concatenate( 1971 [np.array([0]), (ann == i).astype(int), np.array([0])] 1972 ) 1973 ) 1974 == -1 1975 )[0] 1976 result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][ 1977 self.behaviors_dict()[i] 1978 ] = [[start, end, 0] for start, end in zip(starts, ends)] 1979 if self.task_n != 3: 1980 for seq_name, seq_dict in data.items(): 1981 if "annotations" not in seq_dict: 1982 return None 1983 behaviors = np.unique(seq_dict["annotations"]) 1984 ann = seq_dict["annotations"] 1985 key = f'{mode}--{seq_name.split("/")[-1]}' 1986 result[key] = {"mouse1+mouse2": {}} 1987 for i in behaviors: 1988 starts = np.where( 1989 np.diff( 1990 np.concatenate( 1991 [np.array([0]), (ann == i).astype(int), np.array([0])] 1992 ) 1993 ) 1994 == 1 1995 )[0] 1996 ends = np.where( 1997 np.diff( 1998 np.concatenate( 1999 [np.array([0]), (ann == i).astype(int), np.array([0])] 2000 ) 2001 ) 2002 == -1 2003 )[0] 2004 result[key]["mouse1+mouse2"][self.behaviors_dict()[i]] = [ 2005 [start, end, 0] for start, end in zip(starts, ends) 2006 ] 2007 return result
CalMS21 annotation data
Use the 'random:test_from_name:{name}'
and 'val-from-name:{val_name}:test-from-name:{test_name}'
partitioning methods with 'train'
, 'test'
and 'unlabeled'
names to separate into train, test and validation
subsets according to the original files. For example, with 'val-from-name:test:test-from-name:unlabeled'
the data from the test file will go into validation and the unlabeled files will be the test.
Assumes the following file structure:
annotation_path
├── calms21_task_train.npy
├── calms21_task_test.npy
├── calms21_unlabeled_videos_part1.npy
├── calms21_unlabeled_videos_part2.npy
└── calms21_unlabeled_videos_part3.npy
1761 def __init__( 1762 self, 1763 task_n: int = 1, 1764 include_task1: bool = True, 1765 video_order: List = None, 1766 min_frames: Dict = None, 1767 max_frames: Dict = None, 1768 len_segment: int = 128, 1769 overlap: int = 0, 1770 ignored_classes: Set = None, 1771 annotation_path: Union[Set, str] = None, 1772 key_objects: Tuple = None, 1773 treba_files: bool = False, 1774 *args, 1775 **kwargs, 1776 ) -> None: 1777 """ 1778 1779 Parameters 1780 ---------- 1781 task_n : [1, 2] 1782 the number of the task 1783 include_task1 : bool, default True 1784 include task 1 data to training set 1785 video_order : list, optional 1786 a list of video ids that should be processed in the same order (not passed if creating from key objects) 1787 min_frames : dict, optional 1788 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1789 clip start frames (not passed if creating from key objects) 1790 max_frames : dict, optional 1791 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 1792 clip end frames (not passed if creating from key objects) 1793 len_segment : int, default 128 1794 the length of the segments in which the data should be cut (in frames) 1795 overlap : int, default 0 1796 the length of the overlap between neighboring segments (in frames) 1797 ignored_classes : set, optional 1798 the list of behaviors from the behaviors list or file to not annotate 1799 annotation_path : str | set, optional 1800 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 1801 from key objects) 1802 key_objects : tuple, optional 1803 the key objects to load the AnnotationStore from 1804 treba_files : bool, default False 1805 if `True`, TREBA feature files will be loaded 1806 """ 1807 1808 self.task_n = int(task_n) 1809 self.include_task1 = include_task1 1810 if self.task_n == 1: 1811 self.include_task1 = False 1812 self.treba_files = treba_files 1813 if "exclusive" in kwargs: 1814 exclusive = kwargs["exclusive"] 1815 else: 1816 exclusive = True 1817 if "behaviors" in kwargs and kwargs["behaviors"] is not None: 1818 behaviors = kwargs["behaviors"] 1819 else: 1820 behaviors = ["attack", "investigation", "mount", "other"] 1821 if task_n == 3: 1822 exclusive = False 1823 behaviors += [ 1824 "approach", 1825 "disengaged", 1826 "groom", 1827 "intromission", 1828 "mount_attempt", 1829 "sniff_face", 1830 "whiterearing", 1831 ] 1832 super().__init__( 1833 video_order=video_order, 1834 min_frames=min_frames, 1835 max_frames=max_frames, 1836 exclusive=exclusive, 1837 len_segment=len_segment, 1838 overlap=overlap, 1839 behaviors=behaviors, 1840 ignored_classes=ignored_classes, 1841 annotation_path=annotation_path, 1842 key_objects=key_objects, 1843 filter_annotated=False, 1844 interactive=True, 1845 )
Parameters
task_n : [1, 2]
the number of the task
include_task1 : bool, default True
include task 1 data to training set
video_order : list, optional
a list of video ids that should be processed in the same order (not passed if creating from key objects)
min_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip start frames (not passed if creating from key objects)
max_frames : dict, optional
a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
clip end frames (not passed if creating from key objects)
len_segment : int, default 128
the length of the segments in which the data should be cut (in frames)
overlap : int, default 0
the length of the overlap between neighboring segments (in frames)
ignored_classes : set, optional
the list of behaviors from the behaviors list or file to not annotate
annotation_path : str | set, optional
the path or the set of paths to the folder where the annotation files are stored (not passed if creating
from key objects)
key_objects : tuple, optional
the key objects to load the AnnotationStore from
treba_files : bool, default False
if True
, TREBA feature files will be loaded
1847 @classmethod 1848 def get_file_ids( 1849 cls, 1850 task_n: int = 1, 1851 include_task1: bool = False, 1852 treba_files: bool = False, 1853 annotation_path: Union[str, Set] = None, 1854 file_paths=None, 1855 *args, 1856 **kwargs, 1857 ) -> Iterable: 1858 """ 1859 Process data parameters and return a list of ids of the videos that should 1860 be processed by the __init__ function 1861 1862 Parameters 1863 ---------- 1864 task_n : {1, 2, 3} 1865 the index of the CalMS21 challenge task 1866 include_task1 : bool, default False 1867 if `True`, the training file of the task 1 will be loaded 1868 treba_files : bool, default False 1869 if `True`, the TREBA feature files will be loaded 1870 filenames : set, optional 1871 a set of string filenames to search for (only basenames, not the whole paths) 1872 annotation_path : str | set, optional 1873 the path to the folder where the pose and feature files are stored or a set of such paths 1874 (not passed if creating from key objects or from `file_paths`) 1875 file_paths : set, optional 1876 a set of string paths to the pose and feature files 1877 (not passed if creating from key objects or from `data_path`) 1878 1879 Returns 1880 ------- 1881 video_ids : list 1882 a list of video file ids 1883 """ 1884 1885 task_n = int(task_n) 1886 if task_n == 1: 1887 include_task1 = False 1888 files = [] 1889 if treba_files: 1890 postfix = "_features" 1891 else: 1892 postfix = "" 1893 files.append(f"calms21_task{task_n}_train{postfix}.npy") 1894 files.append(f"calms21_task{task_n}_test{postfix}.npy") 1895 if include_task1: 1896 files.append(f"calms21_task1_train{postfix}.npy") 1897 filenames = set(files) 1898 return SequenceAnnotationStore.get_file_ids( 1899 filenames, annotation_path=annotation_path 1900 )
Process data parameters and return a list of ids of the videos that should be processed by the __init__ function
Parameters
task_n : {1, 2, 3}
the index of the CalMS21 challenge task
include_task1 : bool, default False
if True
, the training file of the task 1 will be loaded
treba_files : bool, default False
if True
, the TREBA feature files will be loaded
filenames : set, optional
a set of string filenames to search for (only basenames, not the whole paths)
annotation_path : str | set, optional
the path to the folder where the pose and feature files are stored or a set of such paths
(not passed if creating from key objects or from file_paths
)
file_paths : set, optional
a set of string paths to the pose and feature files
(not passed if creating from key objects or from data_path
)
Returns
video_ids : list a list of video file ids
Inherited Members
2010class CSVAnnotationStore(FileAnnotationStore): # + 2011 """ 2012 CSV type annotation data 2013 2014 Assumes that files are saved as .csv tables with at least the following columns: 2015 - from / start : start of action, 2016 - to / end : end of action, 2017 - class / behavior / behaviour / label / type : action label. 2018 2019 If the times are set in seconds instead of frames, don't forget to set the `fps` parameter to your frame rate. 2020 2021 Assumes the following file structure: 2022 ``` 2023 annotation_path 2024 ├── video1_annotation.csv 2025 └── video2_labels.csv 2026 ``` 2027 Here `annotation_suffix` is `{'_annotation.csv', '_labels.csv'}`. 2028 """ 2029 2030 def __init__( 2031 self, 2032 video_order: List = None, 2033 min_frames: Dict = None, 2034 max_frames: Dict = None, 2035 visibility: Dict = None, 2036 exclusive: bool = True, 2037 len_segment: int = 128, 2038 overlap: int = 0, 2039 behaviors: Set = None, 2040 ignored_classes: Set = None, 2041 annotation_suffix: Union[Set, str] = None, 2042 annotation_path: Union[Set, str] = None, 2043 behavior_file: str = None, 2044 correction: Dict = None, 2045 frame_limit: int = 0, 2046 filter_annotated: bool = False, 2047 filter_background: bool = False, 2048 error_class: str = None, 2049 min_frames_action: int = None, 2050 key_objects: Tuple = None, 2051 visibility_min_score: float = 0.2, 2052 visibility_min_frac: float = 0.7, 2053 mask: Dict = None, 2054 default_agent_name: str = "ind0", 2055 separator: str = ",", 2056 fps: int = 30, 2057 *args, 2058 **kwargs, 2059 ) -> None: 2060 """ 2061 Parameters 2062 ---------- 2063 video_order : list, optional 2064 a list of video ids that should be processed in the same order (not passed if creating from key objects) 2065 min_frames : dict, optional 2066 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2067 clip start frames (not passed if creating from key objects) 2068 max_frames : dict, optional 2069 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2070 clip end frames (not passed if creating from key objects) 2071 visibility : dict, optional 2072 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2073 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 2074 exclusive : bool, default True 2075 if True, the annotation is single-label; if False, multi-label 2076 len_segment : int, default 128 2077 the length of the segments in which the data should be cut (in frames) 2078 overlap : int, default 0 2079 the length of the overlap between neighboring segments (in frames) 2080 behaviors : set, optional 2081 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 2082 loaded from a file) 2083 ignored_classes : set, optional 2084 the list of behaviors from the behaviors list or file to not annotate 2085 annotation_suffix : str | set, optional 2086 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 2087 (not passed if creating from key objects or if irrelevant for the dataset) 2088 annotation_path : str | set, optional 2089 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 2090 from key objects) 2091 behavior_file : str, optional 2092 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 2093 correction : dict, optional 2094 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 2095 can be used to correct for variations in naming or to merge several labels in one 2096 frame_limit : int, default 0 2097 the smallest possible length of a clip (shorter clips are discarded) 2098 filter_annotated : bool, default False 2099 if True, the samples that do not have any labels will be filtered 2100 filter_background : bool, default False 2101 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 2102 error_class : str, optional 2103 the name of the error class (the annotations that intersect with this label will be discarded) 2104 min_frames_action : int, default 0 2105 the minimum length of an action (shorter actions are not annotated) 2106 key_objects : tuple, optional 2107 the key objects to load the AnnotationStore from 2108 visibility_min_score : float, default 5 2109 the minimum visibility score for visibility filtering 2110 visibility_min_frac : float, default 0.7 2111 the minimum fraction of visible frames for visibility filtering 2112 mask : dict, optional 2113 a masked value dictionary (for active learning simulation experiments) 2114 default_agent_name : str, default "ind0" 2115 the clip id to use when there is no given 2116 separator : str, default "," 2117 the separator in the csv files 2118 fps : int, default 30 2119 frames per second in the videos 2120 """ 2121 2122 self.default_agent_name = default_agent_name 2123 self.separator = separator 2124 self.fps = fps 2125 super().__init__( 2126 video_order=video_order, 2127 min_frames=min_frames, 2128 max_frames=max_frames, 2129 visibility=visibility, 2130 exclusive=exclusive, 2131 len_segment=len_segment, 2132 overlap=overlap, 2133 behaviors=behaviors, 2134 ignored_classes=ignored_classes, 2135 ignored_clips=None, 2136 annotation_suffix=annotation_suffix, 2137 annotation_path=annotation_path, 2138 behavior_file=behavior_file, 2139 correction=correction, 2140 frame_limit=frame_limit, 2141 filter_annotated=filter_annotated, 2142 filter_background=filter_background, 2143 error_class=error_class, 2144 min_frames_action=min_frames_action, 2145 key_objects=key_objects, 2146 visibility_min_score=visibility_min_score, 2147 visibility_min_frac=visibility_min_frac, 2148 mask=mask, 2149 ) 2150 2151 def _open_annotations(self, filename: str) -> Dict: 2152 """ 2153 Load the annotation from `filename` 2154 """ 2155 2156 # try: 2157 data = pd.read_csv(filename, sep=self.separator) 2158 data.columns = list(map(lambda x: x.lower(), data.columns)) 2159 starts, ends, actions = None, None, None 2160 start_names = ["from", "start"] 2161 for x in start_names: 2162 if x in data.columns: 2163 starts = data[x] 2164 end_names = ["to", "end"] 2165 for x in end_names: 2166 if x in data.columns: 2167 ends = data[x] 2168 class_names = ["class", "behavior", "behaviour", "type", "label"] 2169 for x in class_names: 2170 if x in data.columns: 2171 actions = data[x] 2172 if starts is None: 2173 raise ValueError("The file must have a column titled 'from' or 'start'!") 2174 if ends is None: 2175 raise ValueError("The file must have a column titled 'to' or 'end'!") 2176 if actions is None: 2177 raise ValueError( 2178 "The file must have a column titled 'class', 'behavior', 'behaviour', 'type' or 'label'!" 2179 ) 2180 times = defaultdict(lambda: defaultdict(lambda: [])) 2181 for start, end, action in zip(starts, ends, actions): 2182 if any([np.isnan(x) for x in [start, end]]): 2183 continue 2184 times[self.default_agent_name][action].append( 2185 [int(start * self.fps), int(end * self.fps), 0] 2186 ) 2187 return times
CSV type annotation data
Assumes that files are saved as .csv tables with at least the following columns:
- from / start : start of action,
- to / end : end of action,
- class / behavior / behaviour / label / type : action label.
If the times are set in seconds instead of frames, don't forget to set the fps
parameter to your frame rate.
Assumes the following file structure:
annotation_path
├── video1_annotation.csv
└── video2_labels.csv
Here annotation_suffix
is {'_annotation.csv', '_labels.csv'}
.
2030 def __init__( 2031 self, 2032 video_order: List = None, 2033 min_frames: Dict = None, 2034 max_frames: Dict = None, 2035 visibility: Dict = None, 2036 exclusive: bool = True, 2037 len_segment: int = 128, 2038 overlap: int = 0, 2039 behaviors: Set = None, 2040 ignored_classes: Set = None, 2041 annotation_suffix: Union[Set, str] = None, 2042 annotation_path: Union[Set, str] = None, 2043 behavior_file: str = None, 2044 correction: Dict = None, 2045 frame_limit: int = 0, 2046 filter_annotated: bool = False, 2047 filter_background: bool = False, 2048 error_class: str = None, 2049 min_frames_action: int = None, 2050 key_objects: Tuple = None, 2051 visibility_min_score: float = 0.2, 2052 visibility_min_frac: float = 0.7, 2053 mask: Dict = None, 2054 default_agent_name: str = "ind0", 2055 separator: str = ",", 2056 fps: int = 30, 2057 *args, 2058 **kwargs, 2059 ) -> None: 2060 """ 2061 Parameters 2062 ---------- 2063 video_order : list, optional 2064 a list of video ids that should be processed in the same order (not passed if creating from key objects) 2065 min_frames : dict, optional 2066 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2067 clip start frames (not passed if creating from key objects) 2068 max_frames : dict, optional 2069 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2070 clip end frames (not passed if creating from key objects) 2071 visibility : dict, optional 2072 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2073 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 2074 exclusive : bool, default True 2075 if True, the annotation is single-label; if False, multi-label 2076 len_segment : int, default 128 2077 the length of the segments in which the data should be cut (in frames) 2078 overlap : int, default 0 2079 the length of the overlap between neighboring segments (in frames) 2080 behaviors : set, optional 2081 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 2082 loaded from a file) 2083 ignored_classes : set, optional 2084 the list of behaviors from the behaviors list or file to not annotate 2085 annotation_suffix : str | set, optional 2086 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 2087 (not passed if creating from key objects or if irrelevant for the dataset) 2088 annotation_path : str | set, optional 2089 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 2090 from key objects) 2091 behavior_file : str, optional 2092 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 2093 correction : dict, optional 2094 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 2095 can be used to correct for variations in naming or to merge several labels in one 2096 frame_limit : int, default 0 2097 the smallest possible length of a clip (shorter clips are discarded) 2098 filter_annotated : bool, default False 2099 if True, the samples that do not have any labels will be filtered 2100 filter_background : bool, default False 2101 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 2102 error_class : str, optional 2103 the name of the error class (the annotations that intersect with this label will be discarded) 2104 min_frames_action : int, default 0 2105 the minimum length of an action (shorter actions are not annotated) 2106 key_objects : tuple, optional 2107 the key objects to load the AnnotationStore from 2108 visibility_min_score : float, default 5 2109 the minimum visibility score for visibility filtering 2110 visibility_min_frac : float, default 0.7 2111 the minimum fraction of visible frames for visibility filtering 2112 mask : dict, optional 2113 a masked value dictionary (for active learning simulation experiments) 2114 default_agent_name : str, default "ind0" 2115 the clip id to use when there is no given 2116 separator : str, default "," 2117 the separator in the csv files 2118 fps : int, default 30 2119 frames per second in the videos 2120 """ 2121 2122 self.default_agent_name = default_agent_name 2123 self.separator = separator 2124 self.fps = fps 2125 super().__init__( 2126 video_order=video_order, 2127 min_frames=min_frames, 2128 max_frames=max_frames, 2129 visibility=visibility, 2130 exclusive=exclusive, 2131 len_segment=len_segment, 2132 overlap=overlap, 2133 behaviors=behaviors, 2134 ignored_classes=ignored_classes, 2135 ignored_clips=None, 2136 annotation_suffix=annotation_suffix, 2137 annotation_path=annotation_path, 2138 behavior_file=behavior_file, 2139 correction=correction, 2140 frame_limit=frame_limit, 2141 filter_annotated=filter_annotated, 2142 filter_background=filter_background, 2143 error_class=error_class, 2144 min_frames_action=min_frames_action, 2145 key_objects=key_objects, 2146 visibility_min_score=visibility_min_score, 2147 visibility_min_frac=visibility_min_frac, 2148 mask=mask, 2149 )
Parameters
video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) behaviors : set, optional the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are loaded from a file) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} (not passed if creating from key objects or if irrelevant for the dataset) annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one frame_limit : int, default 0 the smallest possible length of a clip (shorter clips are discarded) filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) default_agent_name : str, default "ind0" the clip id to use when there is no given separator : str, default "," the separator in the csv files fps : int, default 30 frames per second in the videos
Inherited Members
2190class SIMBAAnnotationStore(FileAnnotationStore): # + 2191 """ 2192 SIMBA paper format data 2193 2194 Assumes the following file structure: 2195 ``` 2196 annotation_path 2197 ├── Video1.csv 2198 ... 2199 └── Video9.csv 2200 """ 2201 2202 def __init__( 2203 self, 2204 video_order: List = None, 2205 min_frames: Dict = None, 2206 max_frames: Dict = None, 2207 visibility: Dict = None, 2208 exclusive: bool = True, 2209 len_segment: int = 128, 2210 overlap: int = 0, 2211 behaviors: Set = None, 2212 ignored_classes: Set = None, 2213 ignored_clips: Set = None, 2214 annotation_path: Union[Set, str] = None, 2215 correction: Dict = None, 2216 filter_annotated: bool = False, 2217 filter_background: bool = False, 2218 error_class: str = None, 2219 min_frames_action: int = None, 2220 key_objects: Tuple = None, 2221 visibility_min_score: float = 0.2, 2222 visibility_min_frac: float = 0.7, 2223 mask: Dict = None, 2224 use_hard_negatives: bool = False, 2225 annotation_suffix: str = None, 2226 *args, 2227 **kwargs, 2228 ) -> None: 2229 """ 2230 Parameters 2231 ---------- 2232 video_order : list, optional 2233 a list of video ids that should be processed in the same order (not passed if creating from key objects) 2234 min_frames : dict, optional 2235 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2236 clip start frames (not passed if creating from key objects) 2237 max_frames : dict, optional 2238 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2239 clip end frames (not passed if creating from key objects) 2240 visibility : dict, optional 2241 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2242 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 2243 exclusive : bool, default True 2244 if True, the annotation is single-label; if False, multi-label 2245 len_segment : int, default 128 2246 the length of the segments in which the data should be cut (in frames) 2247 overlap : int, default 0 2248 the length of the overlap between neighboring segments (in frames) 2249 behaviors : set, optional 2250 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 2251 loaded from a file) 2252 ignored_classes : set, optional 2253 the list of behaviors from the behaviors list or file to not annotate 2254 ignored_clips : set, optional 2255 clip ids to ignore 2256 annotation_path : str | set, optional 2257 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 2258 from key objects) 2259 behavior_file : str, optional 2260 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 2261 correction : dict, optional 2262 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 2263 can be used to correct for variations in naming or to merge several labels in one 2264 filter_annotated : bool, default False 2265 if True, the samples that do not have any labels will be filtered 2266 filter_background : bool, default False 2267 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 2268 error_class : str, optional 2269 the name of the error class (the annotations that intersect with this label will be discarded) 2270 min_frames_action : int, default 0 2271 the minimum length of an action (shorter actions are not annotated) 2272 key_objects : tuple, optional 2273 the key objects to load the AnnotationStore from 2274 visibility_min_score : float, default 5 2275 the minimum visibility score for visibility filtering 2276 visibility_min_frac : float, default 0.7 2277 the minimum fraction of visible frames for visibility filtering 2278 mask : dict, optional 2279 a masked value dictionary (for active learning simulation experiments) 2280 use_hard_negatives : bool, default False 2281 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 2282 annotation_suffix : str | set, optional 2283 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 2284 (not passed if creating from key objects or if irrelevant for the dataset) 2285 """ 2286 2287 super().__init__( 2288 video_order=video_order, 2289 min_frames=min_frames, 2290 max_frames=max_frames, 2291 visibility=visibility, 2292 exclusive=exclusive, 2293 len_segment=len_segment, 2294 overlap=overlap, 2295 behaviors=behaviors, 2296 ignored_classes=ignored_classes, 2297 ignored_clips=ignored_clips, 2298 annotation_suffix=annotation_suffix, 2299 annotation_path=annotation_path, 2300 behavior_file=None, 2301 correction=correction, 2302 frame_limit=0, 2303 filter_annotated=filter_annotated, 2304 filter_background=filter_background, 2305 error_class=error_class, 2306 min_frames_action=min_frames_action, 2307 key_objects=key_objects, 2308 visibility_min_score=visibility_min_score, 2309 visibility_min_frac=visibility_min_frac, 2310 mask=mask, 2311 use_hard_negatives=use_hard_negatives, 2312 interactive=True, 2313 ) 2314 2315 def _open_annotations(self, filename: str) -> Dict: 2316 """ 2317 Load the annotation from filename 2318 2319 Parameters 2320 ---------- 2321 filename : str 2322 path to an annotation file 2323 2324 Returns 2325 ------- 2326 times : dict 2327 a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are 2328 lists of (start, end, ambiguity status) lists 2329 """ 2330 2331 data = pd.read_csv(filename) 2332 columns = [x for x in data.columns if x.split("_")[-1] == "x"] 2333 animals = sorted(set([x.split("_")[-2] for x in columns])) 2334 if len(animals) > 2: 2335 raise ValueError( 2336 "SIMBAAnnotationStore is only implemented for files with 1 or 2 animals!" 2337 ) 2338 if len(animals) == 1: 2339 ind = animals[0] 2340 else: 2341 ind = "+".join(animals) 2342 behaviors = [ 2343 "_".join(x.split("_")[:-1]) 2344 for x in data.columns 2345 if x.split("_")[-1] == "prediction" 2346 ] 2347 result = {} 2348 for behavior in behaviors: 2349 ann = data[f"{behavior}_prediction"].values 2350 diff = np.diff( 2351 np.concatenate([np.array([0]), (ann == 1).astype(int), np.array([0])]) 2352 ) 2353 starts = np.where(diff == 1)[0] 2354 ends = np.where(diff == -1)[0] 2355 result[behavior] = [[start, end, 0] for start, end in zip(starts, ends)] 2356 diff = np.diff( 2357 np.concatenate( 2358 [np.array([0]), (np.isnan(ann)).astype(int), np.array([0])] 2359 ) 2360 ) 2361 starts = np.where(diff == 1)[0] 2362 ends = np.where(diff == -1)[0] 2363 result[f"unknown {behavior}"] = [ 2364 [start, end, 0] for start, end in zip(starts, ends) 2365 ] 2366 if self.behaviors is not None: 2367 for behavior in self.behaviors: 2368 if behavior not in behaviors: 2369 result[f"unknown {behavior}"] = [[0, len(data), 0]] 2370 return {ind: result}
SIMBA paper format data
Assumes the following file structure: ``` annotation_path ├── Video1.csv ... └── Video9.csv
2202 def __init__( 2203 self, 2204 video_order: List = None, 2205 min_frames: Dict = None, 2206 max_frames: Dict = None, 2207 visibility: Dict = None, 2208 exclusive: bool = True, 2209 len_segment: int = 128, 2210 overlap: int = 0, 2211 behaviors: Set = None, 2212 ignored_classes: Set = None, 2213 ignored_clips: Set = None, 2214 annotation_path: Union[Set, str] = None, 2215 correction: Dict = None, 2216 filter_annotated: bool = False, 2217 filter_background: bool = False, 2218 error_class: str = None, 2219 min_frames_action: int = None, 2220 key_objects: Tuple = None, 2221 visibility_min_score: float = 0.2, 2222 visibility_min_frac: float = 0.7, 2223 mask: Dict = None, 2224 use_hard_negatives: bool = False, 2225 annotation_suffix: str = None, 2226 *args, 2227 **kwargs, 2228 ) -> None: 2229 """ 2230 Parameters 2231 ---------- 2232 video_order : list, optional 2233 a list of video ids that should be processed in the same order (not passed if creating from key objects) 2234 min_frames : dict, optional 2235 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2236 clip start frames (not passed if creating from key objects) 2237 max_frames : dict, optional 2238 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2239 clip end frames (not passed if creating from key objects) 2240 visibility : dict, optional 2241 a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are 2242 visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) 2243 exclusive : bool, default True 2244 if True, the annotation is single-label; if False, multi-label 2245 len_segment : int, default 128 2246 the length of the segments in which the data should be cut (in frames) 2247 overlap : int, default 0 2248 the length of the overlap between neighboring segments (in frames) 2249 behaviors : set, optional 2250 the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are 2251 loaded from a file) 2252 ignored_classes : set, optional 2253 the list of behaviors from the behaviors list or file to not annotate 2254 ignored_clips : set, optional 2255 clip ids to ignore 2256 annotation_path : str | set, optional 2257 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 2258 from key objects) 2259 behavior_file : str, optional 2260 the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) 2261 correction : dict, optional 2262 a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, 2263 can be used to correct for variations in naming or to merge several labels in one 2264 filter_annotated : bool, default False 2265 if True, the samples that do not have any labels will be filtered 2266 filter_background : bool, default False 2267 if True, only the unlabeled frames that are close to annotated frames will be labeled as background 2268 error_class : str, optional 2269 the name of the error class (the annotations that intersect with this label will be discarded) 2270 min_frames_action : int, default 0 2271 the minimum length of an action (shorter actions are not annotated) 2272 key_objects : tuple, optional 2273 the key objects to load the AnnotationStore from 2274 visibility_min_score : float, default 5 2275 the minimum visibility score for visibility filtering 2276 visibility_min_frac : float, default 0.7 2277 the minimum fraction of visible frames for visibility filtering 2278 mask : dict, optional 2279 a masked value dictionary (for active learning simulation experiments) 2280 use_hard_negatives : bool, default False 2281 mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing 2282 annotation_suffix : str | set, optional 2283 the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} 2284 (not passed if creating from key objects or if irrelevant for the dataset) 2285 """ 2286 2287 super().__init__( 2288 video_order=video_order, 2289 min_frames=min_frames, 2290 max_frames=max_frames, 2291 visibility=visibility, 2292 exclusive=exclusive, 2293 len_segment=len_segment, 2294 overlap=overlap, 2295 behaviors=behaviors, 2296 ignored_classes=ignored_classes, 2297 ignored_clips=ignored_clips, 2298 annotation_suffix=annotation_suffix, 2299 annotation_path=annotation_path, 2300 behavior_file=None, 2301 correction=correction, 2302 frame_limit=0, 2303 filter_annotated=filter_annotated, 2304 filter_background=filter_background, 2305 error_class=error_class, 2306 min_frames_action=min_frames_action, 2307 key_objects=key_objects, 2308 visibility_min_score=visibility_min_score, 2309 visibility_min_frac=visibility_min_frac, 2310 mask=mask, 2311 use_hard_negatives=use_hard_negatives, 2312 interactive=True, 2313 )
Parameters
video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) behaviors : set, optional the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are loaded from a file) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate ignored_clips : set, optional clip ids to ignore annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) use_hard_negatives : bool, default False mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} (not passed if creating from key objects or if irrelevant for the dataset)