dlc2action.data.annotation_store

Specific implementations of dlc2action.data.base_store.AnnotationStore are defined here

   1#
   2# Copyright 2020-2022 by A. Mathis Group and contributors. All rights reserved.
   3#
   4# This project and all its files are licensed under GNU AGPLv3 or later version. A copy is included in dlc2action/LICENSE.AGPL.
   5#
   6"""
   7Specific implementations of `dlc2action.data.base_store.AnnotationStore` are defined here
   8"""
   9
  10from dlc2action.data.base_store import AnnotationStore
  11from dlc2action.utils import strip_suffix
  12from typing import Dict, List, Tuple, Set, Union
  13import torch
  14import numpy as np
  15import pickle
  16from copy import copy
  17from collections import Counter
  18from collections.abc import Iterable
  19from abc import abstractmethod
  20import os
  21from tqdm import tqdm
  22import pandas as pd
  23from collections import defaultdict
  24from itertools import combinations
  25
  26
  27class EmptyAnnotationStore(AnnotationStore):
  28    def __init__(
  29        self, video_order: List = None, key_objects: Tuple = None, *args, **kwargs
  30    ):
  31        """
  32        Parameters
  33        ----------
  34        video_order : list, optional
  35            a list of video ids that should be processed in the same order (not passed if creating from key objects)
  36        key_objects : tuple, optional
  37            a tuple of key objects
  38        """
  39
  40        pass
  41
  42    def __len__(self) -> int:
  43        """
  44        Get the number of available samples
  45
  46        Returns
  47        -------
  48        length : int
  49            the number of available samples
  50        """
  51
  52        return 0
  53
  54    def remove(self, indices: List) -> None:
  55        """
  56        Remove the samples corresponding to indices
  57
  58        Parameters
  59        ----------
  60        indices : int
  61            a list of integer indices to remove
  62        """
  63
  64        pass
  65
  66    def key_objects(self) -> Tuple:
  67        """
  68        Return a tuple of the key objects necessary to re-create the Store
  69
  70        Returns
  71        -------
  72        key_objects : tuple
  73            a tuple of key objects
  74        """
  75
  76        return ()
  77
  78    def load_from_key_objects(self, key_objects: Tuple) -> None:
  79        """
  80        Load the information from a tuple of key objects
  81
  82        Parameters
  83        ----------
  84        key_objects : tuple
  85            a tuple of key objects
  86        """
  87
  88        pass
  89
  90    def to_ram(self) -> None:
  91        """
  92        Transfer the data samples to RAM if they were previously stored as file paths
  93        """
  94
  95        pass
  96
  97    def get_original_coordinates(self) -> np.ndarray:
  98        """
  99        Return the original coordinates array
 100
 101        Returns
 102        -------
 103        np.ndarray
 104            an array that contains the coordinates of the data samples in original input data (video id, clip id,
 105            start frame)
 106        """
 107
 108        return None
 109
 110    def create_subsample(self, indices: List, ssl_indices: List = None):
 111        """
 112        Create a new store that contains a subsample of the data
 113
 114        Parameters
 115        ----------
 116        indices : list
 117            the indices to be included in the subsample
 118        ssl_indices : list, optional
 119            the indices to be included in the subsample without the annotation data
 120        """
 121
 122        return self.new()
 123
 124    @classmethod
 125    def get_file_ids(cls, *args, **kwargs) -> List:
 126        """
 127        Process data parameters and return a list of ids  of the videos that should
 128        be processed by the __init__ function
 129
 130        Returns
 131        -------
 132        video_ids : list
 133            a list of video file ids
 134        """
 135
 136        return None
 137
 138    def __getitem__(self, ind: int) -> torch.Tensor:
 139        """
 140        Return the annotation of the sample corresponding to an index
 141
 142        Parameters
 143        ----------
 144        ind : int
 145            index of the sample
 146
 147        Returns
 148        -------
 149        sample : torch.Tensor
 150            the corresponding annotation tensor
 151        """
 152
 153        return torch.tensor(float("nan"))
 154
 155    def get_len(self, return_unlabeled: bool) -> int:
 156        """
 157        Get the length of the subsample of labeled/unlabeled data
 158
 159        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
 160        and if return_unlabeled is None the index is already correct
 161
 162        Parameters
 163        ----------
 164        return_unlabeled : bool
 165            the identifier for the subsample
 166
 167        Returns
 168        -------
 169        length : int
 170            the length of the subsample
 171        """
 172
 173        return None
 174
 175    def get_idx(self, index: int, return_unlabeled: bool) -> int:
 176        """
 177        Convert from an index in the subsample of labeled/unlabeled data to an index in the full array
 178
 179        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
 180        and if return_unlabeled is None the index is already correct
 181
 182        Parameters
 183        ----------
 184        index : int
 185            the index in the subsample
 186        return_unlabeled : bool
 187            the identifier for the subsample
 188
 189        Returns
 190        -------
 191        corrected_index : int
 192            the index in the full dataset
 193        """
 194
 195        return index
 196
 197    def count_classes(
 198        self, frac: bool = False, zeros: bool = False, bouts: bool = False
 199    ) -> Dict:
 200        """
 201        Get a dictionary with class-wise frame counts
 202
 203        Parameters
 204        ----------
 205        frac : bool, default False
 206            if True, a fraction of the total frame count is returned
 207
 208        Returns
 209        -------
 210        count_dictionary : dict
 211            a dictionary with class indices as keys and frame counts as values
 212        """
 213
 214        return {}
 215
 216    def behaviors_dict(self) -> Dict:
 217        """
 218        Get a dictionary of class names
 219
 220        Returns
 221        -------
 222        behavior_dictionary: dict
 223            a dictionary with class indices as keys and class names as values
 224        """
 225
 226        return {}
 227
 228    def annotation_class(self) -> str:
 229        """
 230        Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon)
 231
 232        Returns
 233        -------
 234        annotation_class : str
 235            the type of annotation
 236        """
 237
 238        return "none"
 239
 240    def size(self) -> int:
 241        """
 242        Get the total number of frames in the data
 243
 244        Returns
 245        -------
 246        size : int
 247            the total number of frames
 248        """
 249
 250        return None
 251
 252    def filtered_indices(self) -> List:
 253        """
 254        Return the indices of the samples that should be removed
 255
 256        Choosing the indices can be based on any kind of filering defined in the __init__ function by the data
 257        parameters
 258
 259        Returns
 260        -------
 261        indices_to_remove : list
 262            a list of integer indices that should be removed
 263        """
 264
 265        return []
 266
 267    def set_pseudo_labels(self, labels: torch.Tensor) -> None:
 268        """
 269        Set pseudo labels to the unlabeled data
 270
 271        Parameters
 272        ----------
 273        labels : torch.Tensor
 274            a tensor of pseudo-labels for the unlabeled data
 275        """
 276
 277        pass
 278
 279
 280class ActionSegmentationStore(AnnotationStore):  # +
 281    """
 282    A general realization of an annotation store for action segmentation tasks
 283
 284    Assumes the following file structure:
 285    ```
 286    annotation_path
 287    ├── video1_annotation.pickle
 288    └── video2_labels.pickle
 289    ```
 290    Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`.
 291    """
 292
 293    def __init__(
 294        self,
 295        video_order: List = None,
 296        min_frames: Dict = None,
 297        max_frames: Dict = None,
 298        visibility: Dict = None,
 299        exclusive: bool = True,
 300        len_segment: int = 128,
 301        overlap: int = 0,
 302        behaviors: Set = None,
 303        ignored_classes: Set = None,
 304        ignored_clips: Set = None,
 305        annotation_suffix: Union[Set, str] = None,
 306        annotation_path: Union[Set, str] = None,
 307        behavior_file: str = None,
 308        correction: Dict = None,
 309        frame_limit: int = 0,
 310        filter_annotated: bool = False,
 311        filter_background: bool = False,
 312        error_class: str = None,
 313        min_frames_action: int = None,
 314        key_objects: Tuple = None,
 315        visibility_min_score: float = 0.2,
 316        visibility_min_frac: float = 0.7,
 317        mask: Dict = None,
 318        use_hard_negatives: bool = False,
 319        interactive: bool = False,
 320        *args,
 321        **kwargs,
 322    ) -> None:
 323        """
 324        Parameters
 325        ----------
 326        video_order : list, optional
 327            a list of video ids that should be processed in the same order (not passed if creating from key objects)
 328        min_frames : dict, optional
 329            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
 330            clip start frames (not passed if creating from key objects)
 331        max_frames : dict, optional
 332            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
 333            clip end frames (not passed if creating from key objects)
 334        visibility : dict, optional
 335            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
 336            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
 337        exclusive : bool, default True
 338            if True, the annotation is single-label; if False, multi-label
 339        len_segment : int, default 128
 340            the length of the segments in which the data should be cut (in frames)
 341        overlap : int, default 0
 342            the length of the overlap between neighboring segments (in frames)
 343        behaviors : set, optional
 344            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
 345            loaded from a file)
 346        ignored_classes : set, optional
 347            the list of behaviors from the behaviors list or file to not annotate
 348        ignored_clips : set, optional
 349            clip ids to ignore
 350        annotation_suffix : str | set, optional
 351            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
 352            (not passed if creating from key objects or if irrelevant for the dataset)
 353        annotation_path : str | set, optional
 354            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
 355            from key objects)
 356        behavior_file : str, optional
 357            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
 358        correction : dict, optional
 359            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
 360            can be used to correct for variations in naming or to merge several labels in one
 361        frame_limit : int, default 0
 362            the smallest possible length of a clip (shorter clips are discarded)
 363        filter_annotated : bool, default False
 364            if True, the samples that do not have any labels will be filtered
 365        filter_background : bool, default False
 366            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
 367        error_class : str, optional
 368            the name of the error class (the annotations that intersect with this label will be discarded)
 369        min_frames_action : int, default 0
 370            the minimum length of an action (shorter actions are not annotated)
 371        key_objects : tuple, optional
 372            the key objects to load the AnnotationStore from
 373        visibility_min_score : float, default 5
 374            the minimum visibility score for visibility filtering
 375        visibility_min_frac : float, default 0.7
 376            the minimum fraction of visible frames for visibility filtering
 377        mask : dict, optional
 378            a masked value dictionary (for active learning simulation experiments)
 379        use_hard_negatives : bool, default False
 380            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
 381        interactive : bool, default False
 382            if `True`, annotation is assigned to pairs of individuals
 383        """
 384
 385        super().__init__()
 386
 387        if ignored_clips is None:
 388            ignored_clips = []
 389        self.len_segment = int(len_segment)
 390        self.exclusive = exclusive
 391        if overlap < 1:
 392            overlap = overlap * len_segment
 393        self.overlap = int(overlap)
 394        self.video_order = video_order
 395        self.min_frames = min_frames
 396        self.max_frames = max_frames
 397        self.visibility = visibility
 398        self.vis_min_score = visibility_min_score
 399        self.vis_min_frac = visibility_min_frac
 400        self.mask = mask
 401        self.use_negatives = use_hard_negatives
 402        self.interactive = interactive
 403        self.ignored_clips = ignored_clips
 404        self.file_paths = self._get_file_paths(annotation_path)
 405        self.ignored_classes = ignored_classes
 406        self.update_behaviors = False
 407
 408        self.ram = True
 409        self.original_coordinates = []
 410        self.filtered = []
 411
 412        self.step = self.len_segment - self.overlap
 413
 414        self.ann_suffix = annotation_suffix
 415        self.annotation_folder = annotation_path
 416        self.filter_annotated = filter_annotated
 417        self.filter_background = filter_background
 418        self.frame_limit = frame_limit
 419        self.min_frames_action = min_frames_action
 420        self.error_class = error_class
 421
 422        if correction is None:
 423            correction = {}
 424        self.correction = correction
 425
 426        if self.max_frames is None:
 427            self.max_frames = defaultdict(lambda: {})
 428        if self.min_frames is None:
 429            self.min_frames = defaultdict(lambda: {})
 430
 431        lists = [self.annotation_folder, self.ann_suffix]
 432        for i in range(len(lists)):
 433            iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str))
 434            if lists[i] is not None:
 435                if not iterable:
 436                    lists[i] = [lists[i]]
 437                lists[i] = [x for x in lists[i]]
 438        self.annotation_folder, self.ann_suffix = lists
 439
 440        if ignored_classes is None:
 441            ignored_classes = []
 442        self.ignored_classes = ignored_classes
 443        self._set_behaviors(behaviors, ignored_classes, behavior_file)
 444
 445        if key_objects is None and self.video_order is not None:
 446            self.data = self._load_data()
 447        elif key_objects is not None:
 448            self.load_from_key_objects(key_objects)
 449        else:
 450            self.data = None
 451        self.labeled_indices, self.unlabeled_indices = self._compute_labeled()
 452
 453    def __getitem__(self, ind):
 454        if self.data is None:
 455            raise RuntimeError("The annotation store data has not been initialized!")
 456        return self.data[ind]
 457
 458    def __len__(self) -> int:
 459        if self.data is None:
 460            raise RuntimeError("The annotation store data has not been initialized!")
 461        return len(self.data)
 462
 463    def remove(self, indices: List) -> None:
 464        """
 465        Remove the samples corresponding to indices
 466
 467        Parameters
 468        ----------
 469        indices : list
 470            a list of integer indices to remove
 471        """
 472
 473        if len(indices) > 0:
 474            mask = np.ones(len(self.data))
 475            mask[indices] = 0
 476            mask = mask.astype(bool)
 477            self.data = self.data[mask]
 478            self.original_coordinates = self.original_coordinates[mask]
 479
 480    def key_objects(self) -> Tuple:
 481        """
 482        Return a tuple of the key objects necessary to re-create the Store
 483
 484        Returns
 485        -------
 486        key_objects : tuple
 487            a tuple of key objects
 488        """
 489
 490        return (
 491            self.original_coordinates,
 492            self.data,
 493            self.behaviors,
 494            self.exclusive,
 495            self.len_segment,
 496            self.step,
 497            self.overlap,
 498        )
 499
 500    def load_from_key_objects(self, key_objects: Tuple) -> None:
 501        """
 502        Load the information from a tuple of key objects
 503
 504        Parameters
 505        ----------
 506        key_objects : tuple
 507            a tuple of key objects
 508        """
 509
 510        (
 511            self.original_coordinates,
 512            self.data,
 513            self.behaviors,
 514            self.exclusive,
 515            self.len_segment,
 516            self.step,
 517            self.overlap,
 518        ) = key_objects
 519        self.labeled_indices, self.unlabeled_indices = self._compute_labeled()
 520
 521    def to_ram(self) -> None:
 522        """
 523        Transfer the data samples to RAM if they were previously stored as file paths
 524        """
 525
 526        pass
 527
 528    def get_original_coordinates(self) -> np.ndarray:
 529        """
 530        Return the video_indices array
 531
 532        Returns
 533        -------
 534        original_coordinates : numpy.ndarray
 535            an array that contains the coordinates of the data samples in original input data
 536        """
 537        return self.original_coordinates
 538
 539    def create_subsample(self, indices: List, ssl_indices: List = None):
 540        """
 541        Create a new store that contains a subsample of the data
 542
 543        Parameters
 544        ----------
 545        indices : list
 546            the indices to be included in the subsample
 547        ssl_indices : list, optional
 548            the indices to be included in the subsample without the annotation data
 549        """
 550
 551        if ssl_indices is None:
 552            ssl_indices = []
 553        data = copy(self.data)
 554        data[ssl_indices, ...] = -100
 555        new = self.new()
 556        new.original_coordinates = self.original_coordinates[indices + ssl_indices]
 557        new.data = self.data[indices + ssl_indices]
 558        new.labeled_indices, new.unlabeled_indices = new._compute_labeled()
 559        new.behaviors = self.behaviors
 560        new.exclusive = self.exclusive
 561        new.len_segment = self.len_segment
 562        new.step = self.step
 563        new.overlap = self.overlap
 564        new.max_frames = self.max_frames
 565        new.min_frames = self.min_frames
 566        return new
 567
 568    def get_len(self, return_unlabeled: bool) -> int:
 569        """
 570        Get the length of the subsample of labeled/unlabeled data
 571
 572        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
 573        and if return_unlabeled is None the index is already correct
 574
 575        Parameters
 576        ----------
 577        return_unlabeled : bool
 578            the identifier for the subsample
 579
 580        Returns
 581        -------
 582        length : int
 583            the length of the subsample
 584        """
 585
 586        if self.data is None:
 587            raise RuntimeError("The annotation store data has not been initialized!")
 588        elif return_unlabeled is None:
 589            return len(self.data)
 590        elif return_unlabeled:
 591            return len(self.unlabeled_indices)
 592        else:
 593            return len(self.labeled_indices)
 594
 595    def get_indices(self, return_unlabeled: bool) -> List:
 596        """
 597        Get a list of indices of samples in the labeled/unlabeled subset
 598
 599        Parameters
 600        ----------
 601        return_unlabeled : bool
 602            the identifier for the subsample (`True` for unlabeled, `False` for labeled, `None` for the
 603            whole dataset)
 604
 605        Returns
 606        -------
 607        indices : list
 608            a list of indices that meet the criteria
 609        """
 610
 611        return list(range(len(self.data)))
 612
 613    def count_classes(
 614        self, perc: bool = False, zeros: bool = False, bouts: bool = False
 615    ) -> Dict:
 616        """
 617        Get a dictionary with class-wise frame counts
 618
 619        Parameters
 620        ----------
 621        perc : bool, default False
 622            if `True`, a fraction of the total frame count is returned
 623        zeros : bool, default False
 624            if `True` and annotation is not exclusive, zero counts are returned
 625        bouts : bool, default False
 626            if `True`, instead of frame counts segment counts are returned
 627
 628        Returns
 629        -------
 630        count_dictionary : dict
 631            a dictionary with class indices as keys and frame counts as values
 632        """
 633
 634        if bouts:
 635            if self.overlap != 0:
 636                data = {}
 637                for video, value in self.max_frames.items():
 638                    for clip, end in value.items():
 639                        length = end - self._get_min_frame(video, clip)
 640                        if self.exclusive:
 641                            data[f"{video}---{clip}"] = -100 * torch.ones(length)
 642                        else:
 643                            data[f"{video}---{clip}"] = -100 * torch.ones(
 644                                (len(self.behaviors_dict()), length)
 645                            )
 646                for x, coords in zip(self.data, self.original_coordinates):
 647                    split = coords[0].split("---")
 648                    l = self._get_max_frame(split[0], split[1]) - self._get_min_frame(
 649                        split[0], split[1]
 650                    )
 651                    i = coords[1]
 652                    start = int(i) * self.step
 653                    end = min(start + self.len_segment, l)
 654                    data[coords[0]][..., start:end] = x[..., : end - start]
 655                values = []
 656                for key, value in data.items():
 657                    values.append(value)
 658                    values.append(-100 * torch.ones((*value.shape[:-1], 1)))
 659                data = torch.cat(values, -1).T
 660            else:
 661                data = copy(self.data)
 662                if self.exclusive:
 663                    data = data.flatten()
 664                else:
 665                    data = data.transpose(1, 2).reshape(-1, len(self.behaviors))
 666            count_dictionary = {}
 667            for c in self.behaviors_dict():
 668                if self.exclusive:
 669                    arr = data == c
 670                else:
 671                    if zeros:
 672                        arr = data[:, c] == 0
 673                    else:
 674                        arr = data[:, c] == 1
 675                output, indices = torch.unique_consecutive(arr, return_inverse=True)
 676                true_indices = torch.where(output)[0]
 677                count_dictionary[c] = len(true_indices)
 678        else:
 679            ind = 1
 680            if zeros:
 681                ind = 0
 682            if self.exclusive:
 683                count_dictionary = dict(Counter(self.data.flatten().cpu().numpy()))
 684            else:
 685                d = {}
 686                for i in range(self.data.shape[1]):
 687                    cnt = Counter(self.data[:, i, :].flatten().cpu().numpy())
 688                    d[i] = cnt[ind]
 689                count_dictionary = d
 690            if perc:
 691                total = sum([v for k, v in count_dictionary.items()])
 692                count_dictionary = {k: v / total for k, v in count_dictionary.items()}
 693        for i in self.behaviors_dict():
 694            if i not in count_dictionary:
 695                count_dictionary[i] = 0
 696        return count_dictionary
 697
 698    def behaviors_dict(self) -> Dict:
 699        """
 700        Get a dictionary of class names
 701
 702        Returns
 703        -------
 704        behavior_dictionary: dict
 705            a dictionary with class indices as keys and class names as values
 706        """
 707
 708        # if self.behaviors is None
 709        if self.exclusive and "other" not in self.behaviors:
 710            d = {i + 1: b for i, b in enumerate(self.behaviors)}
 711            d[0] = "other"
 712        else:
 713            d = {i: b for i, b in enumerate(self.behaviors)}
 714        return d
 715
 716    def annotation_class(self) -> str:
 717        """
 718        Get the type of annotation ('exclusive_classification', 'nonexclusive_classification')
 719
 720        Returns
 721        -------
 722        annotation_class : str
 723            the type of annotation
 724        """
 725
 726        if self.exclusive:
 727            return "exclusive_classification"
 728        else:
 729            return "nonexclusive_classification"
 730
 731    def size(self) -> int:
 732        """
 733        Get the total number of frames in the data
 734
 735        Returns
 736        -------
 737        size : int
 738            the total number of frames
 739        """
 740
 741        return self.data.shape[0] * self.data.shape[-1]
 742
 743    def filtered_indices(self) -> List:
 744        """
 745        Return the indices of the samples that should be removed
 746
 747        Choosing the indices can be based on any kind of filering defined in the __init__ function by the data
 748        parameters
 749
 750        Returns
 751        -------
 752        indices_to_remove : list
 753            a list of integer indices that should be removed
 754        """
 755
 756        return self.filtered
 757
 758    def set_pseudo_labels(self, labels: torch.Tensor) -> None:
 759        """
 760        Set pseudo labels to the unlabeled data
 761
 762        Parameters
 763        ----------
 764        labels : torch.Tensor
 765            a tensor of pseudo-labels for the unlabeled data
 766        """
 767
 768        self.data[self.unlabeled_indices] = labels
 769
 770    @classmethod
 771    def get_file_ids(
 772        cls,
 773        annotation_path: Union[str, Set],
 774        annotation_suffix: Union[str, Set],
 775        *args,
 776        **kwargs,
 777    ) -> List:
 778        """
 779        Process data parameters and return a list of ids  of the videos that should
 780        be processed by the __init__ function
 781
 782        Parameters
 783        ----------
 784        annotation_path : str | set
 785            the path or the set of paths to the folder where the annotation files are stored
 786        annotation_suffix : str | set, optional
 787            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
 788
 789        Returns
 790        -------
 791        video_ids : list
 792            a list of video file ids
 793        """
 794
 795        lists = [annotation_path, annotation_suffix]
 796        for i in range(len(lists)):
 797            iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str))
 798            if lists[i] is not None:
 799                if not iterable:
 800                    lists[i] = [lists[i]]
 801                lists[i] = [x for x in lists[i]]
 802        annotation_path, annotation_suffix = lists
 803        files = []
 804        for folder in annotation_path:
 805            files += [
 806                strip_suffix(os.path.basename(file), annotation_suffix)
 807                for file in os.listdir(folder)
 808                if file.endswith(tuple([x for x in annotation_suffix]))
 809            ]
 810        files = sorted(files, key=lambda x: os.path.basename(x))
 811        return files
 812
 813    def _set_behaviors(
 814        self, behaviors: List, ignored_classes: List, behavior_file: str
 815    ):
 816        """
 817        Get a list of behaviors that should be annotated from behavior parameters
 818        """
 819
 820        if behaviors is not None:
 821            for b in ignored_classes:
 822                if b in behaviors:
 823                    behaviors.remove(b)
 824        self.behaviors = behaviors
 825
 826    def _compute_labeled(self) -> Tuple[torch.Tensor, torch.Tensor]:
 827        """
 828        Get the indices of labeled (annotated) and unlabeled samples
 829        """
 830
 831        if self.data is not None and len(self.data) > 0:
 832            unlabeled = torch.sum(self.data != -100, dim=1) == 0
 833            labeled_indices = torch.where(~unlabeled)[0]
 834            unlabeled_indices = torch.where(unlabeled)[0]
 835        else:
 836            labeled_indices, unlabeled_indices = torch.tensor([]), torch.tensor([])
 837        return labeled_indices, unlabeled_indices
 838
 839    def _generate_annotation(self, times: Dict, name: str) -> Dict:
 840        """
 841        Process a loaded annotation file to generate a training labels dictionary
 842        """
 843
 844        annotation = {}
 845        if self.behaviors is None and times is not None:
 846            behaviors = set()
 847            for d in times.values():
 848                behaviors.update([k for k, v in d.items()])
 849            self.behaviors = [
 850                x
 851                for x in sorted(behaviors)
 852                if x not in self.ignored_classes
 853                and not x.startswith("negative")
 854                and not x.startswith("unknown")
 855            ]
 856        elif self.behaviors is None and times is None:
 857            raise ValueError("Cannot generate annotqtion without behavior information!")
 858        beh_inv = {v: k for k, v in self.behaviors_dict().items()}
 859        # if there is no annotation file, generate empty annotation
 860        if self.interactive:
 861            clips = [
 862                "+".join(sorted(x))
 863                for x in combinations(self.max_frames[name].keys(), 2)
 864            ]
 865        else:
 866            clips = list(self.max_frames[name].keys())
 867        if times is None:
 868            clips = [x for x in clips if x not in self.ignored_clips]
 869        # otherwise, apply filters and generate label arrays
 870        else:
 871            clips = [
 872                x
 873                for x in clips
 874                if x not in self.ignored_clips and x not in times.keys()
 875            ]
 876            for ind in times.keys():
 877                try:
 878                    min_frame = self._get_min_frame(name, ind)
 879                    max_frame = self._get_max_frame(name, ind)
 880                except KeyError:
 881                    continue
 882                go_on = max_frame - min_frame + 1 >= self.frame_limit
 883                if go_on:
 884                    v_len = max_frame - min_frame + 1
 885                    if self.exclusive:
 886                        if not self.filter_background:
 887                            value = beh_inv.get("other", 0)
 888                            labels = np.ones(v_len, dtype=np.compat.long) * value
 889                        else:
 890                            labels = -100 * np.ones(v_len, dtype=np.compat.long)
 891                    else:
 892                        labels = np.zeros((len(self.behaviors), v_len), dtype=np.float)
 893                    cat_new = []
 894                    for cat in times[ind].keys():
 895                        if cat.startswith("unknown"):
 896                            cat_new.append(cat)
 897                    for cat in times[ind].keys():
 898                        if cat.startswith("negative"):
 899                            cat_new.append(cat)
 900                    for cat in times[ind].keys():
 901                        if not cat.startswith("negative") and not cat.startswith(
 902                            "unknown"
 903                        ):
 904                            cat_new.append(cat)
 905                    for cat in cat_new:
 906                        neg = False
 907                        unknown = False
 908                        cat_times = times[ind][cat]
 909                        if self.use_negatives and cat.startswith("negative"):
 910                            cat = " ".join(cat.split()[1:])
 911                            neg = True
 912                        elif cat.startswith("unknown"):
 913                            cat = " ".join(cat.split()[1:])
 914                            unknown = True
 915                        if cat in self.correction:
 916                            cat = self.correction[cat]
 917                        for start, end, amb in cat_times:
 918                            if end > self._get_max_frame(name, ind) + 1:
 919                                end = self._get_max_frame(name, ind) + 1
 920                            if amb != 0:
 921                                continue
 922                            start -= min_frame
 923                            end -= min_frame
 924                            if (
 925                                self.min_frames_action is not None
 926                                and end - start < self.min_frames_action
 927                            ):
 928                                continue
 929                            if (
 930                                self.vis_min_frac > 0
 931                                and self.vis_min_score > 0
 932                                and self.visibility is not None
 933                            ):
 934                                s = 0
 935                                for ind_k in ind.split("+"):
 936                                    s += np.sum(
 937                                        self.visibility[name][ind_k][start:end]
 938                                        > self.vis_min_score
 939                                    )
 940                                if s < self.vis_min_frac * (end - start) * len(
 941                                    ind.split("+")
 942                                ):
 943                                    continue
 944                            if cat in beh_inv:
 945                                cat_i_global = beh_inv[cat]
 946                                if self.exclusive:
 947                                    labels[start:end] = cat_i_global
 948                                else:
 949                                    if unknown:
 950                                        labels[cat_i_global, start:end] = -100
 951                                    elif neg:
 952                                        labels[cat_i_global, start:end] = 2
 953                                    else:
 954                                        labels[cat_i_global, start:end] = 1
 955                            else:
 956                                self.not_found.add(cat)
 957                                if self.filter_background:
 958                                    if not self.exclusive:
 959                                        labels[:, start:end][
 960                                            labels[:, start:end] == 0
 961                                        ] = 3
 962                                    else:
 963                                        labels[start:end][labels[start:end] == -100] = 0
 964
 965                    if self.error_class is not None and self.error_class in times[ind]:
 966                        for start, end, amb in times[ind][self.error_class]:
 967                            if self.exclusive:
 968                                labels[start:end] = -100
 969                            else:
 970                                labels[:, start:end] = -100
 971                    annotation[os.path.basename(name) + "---" + str(ind)] = labels
 972        for ind in clips:
 973            try:
 974                min_frame = self._get_min_frame(name, ind)
 975                max_frame = self._get_max_frame(name, ind)
 976            except KeyError:
 977                continue
 978            go_on = max_frame - min_frame + 1 >= self.frame_limit
 979            if go_on:
 980                v_len = max_frame - min_frame + 1
 981                if self.exclusive:
 982                    annotation[
 983                        os.path.basename(name) + "---" + str(ind)
 984                    ] = -100 * np.ones(v_len, dtype=np.compat.long)
 985                else:
 986                    annotation[
 987                        os.path.basename(name) + "---" + str(ind)
 988                    ] = -100 * np.ones((len(self.behaviors), v_len), dtype=np.float)
 989        return annotation
 990
 991    def _make_trimmed_annotations(self, annotations_dict: Dict) -> torch.Tensor:
 992        """
 993        Cut a label dictionary into overlapping pieces of equal length
 994        """
 995
 996        labels = []
 997        self.original_coordinates = []
 998        masked_all = []
 999        for v_id in sorted(annotations_dict.keys()):
1000            if v_id in annotations_dict:
1001                annotations = annotations_dict[v_id]
1002            else:
1003                raise ValueError(
1004                    f'The id list in {v_id.split("---")[0]} is not consistent across files'
1005                )
1006            split = v_id.split("---")
1007            if len(split) > 1:
1008                video_id, ind = split
1009            else:
1010                video_id = split[0]
1011                ind = ""
1012            min_frame = self._get_min_frame(video_id, ind)
1013            max_frame = self._get_max_frame(video_id, ind)
1014            v_len = max_frame - min_frame + 1
1015            sp = np.arange(0, v_len, self.step)
1016            pad = sp[-1] + self.len_segment - v_len
1017            if self.exclusive:
1018                annotations = np.pad(annotations, ((0, pad)), constant_values=-100)
1019            else:
1020                annotations = np.pad(
1021                    annotations, ((0, 0), (0, pad)), constant_values=-100
1022                )
1023            masked = np.zeros(annotations.shape)
1024            if (
1025                self.mask is not None
1026                and video_id in self.mask["masked"]
1027                and ind in self.mask["masked"][video_id]
1028            ):
1029                for start, end in self.mask["masked"][video_id][ind]:
1030                    masked[..., int(start) : int(end)] = 1
1031            for i, start in enumerate(sp):
1032                self.original_coordinates.append((v_id, i))
1033                if self.exclusive:
1034                    ann = annotations[start : start + self.len_segment]
1035                    m = masked[start : start + self.len_segment]
1036                else:
1037                    ann = annotations[:, start : start + self.len_segment]
1038                    m = masked[:, start : start + self.len_segment]
1039                labels.append(ann)
1040                masked_all.append(m)
1041        self.original_coordinates = np.array(self.original_coordinates)
1042        labels = torch.tensor(np.array(labels))
1043        masked_all = torch.tensor(np.array(masked_all)).int().bool()
1044        if self.filter_background and not self.exclusive:
1045            for i, label in enumerate(labels):
1046                label[:, torch.sum((label == 1) | (label == 3), 0) == 0] = -100
1047                label[label == 3] = 0
1048        labels[(labels != -100) & masked_all] = -200
1049        return labels
1050
1051    @classmethod
1052    def _get_file_paths(cls, annotation_path: Union[str, Set]) -> List:
1053        """
1054        Get a list of relevant files
1055        """
1056
1057        file_paths = []
1058        if annotation_path is not None:
1059            if isinstance(annotation_path, str):
1060                annotation_path = [annotation_path]
1061            for folder in annotation_path:
1062                file_paths += [os.path.join(folder, x) for x in os.listdir(folder)]
1063        return file_paths
1064
1065    def _get_max_frame(self, video_id: str, clip_id: str):
1066        """
1067        Get the end frame of a clip in a video
1068        """
1069
1070        if clip_id in self.max_frames[video_id]:
1071            return self.max_frames[video_id][clip_id]
1072        else:
1073            return min(
1074                [self.max_frames[video_id][ind_k] for ind_k in clip_id.split("+")]
1075            )
1076
1077    def _get_min_frame(self, video_id, clip_id):
1078        """
1079        Get the start frame of a clip in a video
1080        """
1081
1082        if clip_id in self.min_frames[video_id]:
1083            return self.min_frames[video_id][clip_id]
1084        else:
1085            return max(
1086                [self.min_frames[video_id][ind_k] for ind_k in clip_id.split("+")]
1087            )
1088
1089    @abstractmethod
1090    def _load_data(self) -> torch.Tensor:
1091        """
1092        Load behavior annotation and generate annotation prompts
1093        """
1094
1095
1096class FileAnnotationStore(ActionSegmentationStore):  # +
1097    """
1098    A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to one video
1099    """
1100
1101    def _generate_max_min_frames(self, times: Dict, video_id: str) -> None:
1102        """
1103        Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore`
1104        """
1105
1106        if video_id in self.max_frames:
1107            return
1108        for ind, cat_dict in times.items():
1109            maxes = []
1110            mins = []
1111            for cat, cat_list in cat_dict.items():
1112                if len(cat_list) > 0:
1113                    maxes.append(max([x[1] for x in cat_list]))
1114                    mins.append(min([x[0] for x in cat_list]))
1115            self.max_frames[video_id][ind] = max(maxes)
1116            self.min_frames[video_id][ind] = min(mins)
1117
1118    def _load_data(self) -> torch.Tensor:
1119        """
1120        Load behavior annotation and generate annotation prompts
1121        """
1122
1123        if self.video_order is None:
1124            return None
1125
1126        files = []
1127        for x in self.video_order:
1128            ok = False
1129            for folder in self.annotation_folder:
1130                for s in self.ann_suffix:
1131                    file = os.path.join(folder, x + s)
1132                    if os.path.exists(file):
1133                        files.append(file)
1134                        ok = True
1135                        break
1136            if not ok:
1137                files.append(None)
1138        self.not_found = set()
1139        annotations_dict = {}
1140        print("Computing annotation arrays...")
1141        for name, filename in tqdm(list(zip(self.video_order, files))):
1142            if filename is not None:
1143                times = self._open_annotations(filename)
1144            else:
1145                times = None
1146            if times is not None:
1147                self._generate_max_min_frames(times, name)
1148            annotations_dict.update(self._generate_annotation(times, name))
1149            del times
1150        annotation = self._make_trimmed_annotations(annotations_dict)
1151        del annotations_dict
1152        if self.filter_annotated:
1153            if self.exclusive:
1154                s = torch.sum((annotation != -100), dim=1)
1155            else:
1156                s = torch.sum(
1157                    torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1
1158                )
1159            self.filtered += torch.where(s == 0)[0].tolist()
1160        annotation[annotation == -200] = -100
1161        return annotation
1162
1163    @abstractmethod
1164    def _open_annotations(self, filename: str) -> Dict:
1165        """
1166        Load the annotation from filename
1167
1168        Parameters
1169        ----------
1170        filename : str
1171            path to an annotation file
1172
1173        Returns
1174        -------
1175        times : dict
1176            a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are
1177            lists of (start, end, ambiguity status) lists
1178        """
1179
1180
1181class SequenceAnnotationStore(ActionSegmentationStore):  # +
1182    """
1183    A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to multiple videos
1184    """
1185
1186    def _generate_max_min_frames(self, times: Dict) -> None:
1187        """
1188        Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore`
1189        """
1190
1191        for video_id in times:
1192            if video_id in self.max_frames:
1193                continue
1194            self.max_frames[video_id] = {}
1195            for ind, cat_dict in times[video_id].items():
1196                maxes = []
1197                mins = []
1198                for cat, cat_list in cat_dict.items():
1199                    maxes.append(max([x[1] for x in cat_list]))
1200                    mins.append(min([x[0] for x in cat_list]))
1201                self.max_frames[video_id][ind] = max(maxes)
1202                self.min_frames[video_id][ind] = min(mins)
1203
1204    @classmethod
1205    def get_file_ids(
1206        cls,
1207        filenames: List = None,
1208        annotation_path: str = None,
1209        *args,
1210        **kwargs,
1211    ) -> List:
1212        """
1213        Process data parameters and return a list of ids  of the videos that should
1214        be processed by the __init__ function
1215
1216        Parameters
1217        ----------
1218        filenames : list, optional
1219            a list of annotation file paths
1220        annotation_path : str, optional
1221            path to the annotation folder
1222
1223        Returns
1224        -------
1225        video_ids : list
1226            a list of video file ids
1227        """
1228
1229        file_paths = []
1230        if annotation_path is not None:
1231            if isinstance(annotation_path, str):
1232                annotation_path = [annotation_path]
1233            file_paths = []
1234            for folder in annotation_path:
1235                file_paths += [os.path.join(folder, x) for x in os.listdir(folder)]
1236        ids = set()
1237        for f in file_paths:
1238            if os.path.basename(f) in filenames:
1239                ids.add(os.path.basename(f))
1240        ids = sorted(ids)
1241        return ids
1242
1243    def _load_data(self) -> torch.Tensor:
1244        """
1245        Load behavior annotation and generate annotation prompts
1246        """
1247
1248        if self.video_order is None:
1249            return None
1250
1251        files = []
1252        for f in self.file_paths:
1253            if os.path.basename(f) in self.video_order:
1254                files.append(f)
1255        self.not_found = set()
1256        annotations_dict = {}
1257        for name, filename in tqdm(zip(self.video_order, files)):
1258            if filename is not None:
1259                times = self._open_sequences(filename)
1260            else:
1261                times = None
1262            if times is not None:
1263                self._generate_max_min_frames(times)
1264                none_ids = []
1265                for video_id, sequence_dict in times.items():
1266                    if sequence_dict is None:
1267                        none_ids.append(sequence_dict)
1268                        continue
1269                    annotations_dict.update(
1270                        self._generate_annotation(sequence_dict, video_id)
1271                    )
1272                for video_id in none_ids:
1273                    annotations_dict.update(self._generate_annotation(None, video_id))
1274                del times
1275        annotation = self._make_trimmed_annotations(annotations_dict)
1276        del annotations_dict
1277        if self.filter_annotated:
1278            if self.exclusive:
1279                s = torch.sum((annotation != -100), dim=1)
1280            else:
1281                s = torch.sum(
1282                    torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1
1283                )
1284            self.filtered += torch.where(s == 0)[0].tolist()
1285        annotation[annotation == -200] = -100
1286        return annotation
1287
1288    @abstractmethod
1289    def _open_sequences(self, filename: str) -> Dict:
1290        """
1291        Load the annotation from filename
1292
1293        Parameters
1294        ----------
1295        filename : str
1296            path to an annotation file
1297
1298        Returns
1299        -------
1300        times : dict
1301            a nested dictionary where first-level keys are video ids, second-level keys are clip ids,
1302            third-level keys are categories and values are
1303            lists of (start, end, ambiguity status) lists
1304        """
1305
1306
1307class DLCAnnotationStore(FileAnnotationStore):  # +
1308    """
1309    DLC type annotation data
1310
1311    The files are either the DLC2Action GUI output or a pickled dictionary of the following structure:
1312        - nested dictionary,
1313        - first-level keys are individual IDs,
1314        - second-level keys are labels,
1315        - values are lists of intervals,
1316        - the lists of intervals is formatted as `[start_frame, end_frame, ambiguity]`,
1317        - ambiguity is 1 if the action is ambiguous (!!at the moment DLC2Action will IGNORE those intervals!!) or 0 if it isn't.
1318
1319    A minimum working example of such a dictionary is:
1320    ```
1321    {
1322        "ind0": {},
1323        "ind1": {
1324            "running": [60, 70, 0]],
1325            "eating": []
1326        }
1327    }
1328    ```
1329
1330    Here there are two animals: `"ind0"` and `"ind1"`, and two actions: running and eating.
1331    The only annotated action is eating for `"ind1"` between frames 60 and 70.
1332
1333    If you generate those files manually, run this code for a sanity check:
1334    ```
1335    import pickle
1336
1337    with open("/path/to/annotation.pickle", "rb") as f:
1338    data = pickle.load(f)
1339
1340    for ind, ind_dict in data.items():
1341        print(f'individual {ind}:')
1342        for label, intervals in ind_dict.items():
1343            for start, end, ambiguity in intervals:
1344                if ambiguity == 0:
1345                    print(f'  from {start} to {end} frame: {label}')
1346    ```
1347
1348    Assumes the following file structure:
1349    ```
1350    annotation_path
1351    ├── video1_annotation.pickle
1352    └── video2_labels.pickle
1353    ```
1354    Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`.
1355    """
1356
1357    def _open_annotations(self, filename: str) -> Dict:
1358        """
1359        Load the annotation from `filename`
1360        """
1361
1362        try:
1363            with open(filename, "rb") as f:
1364                data = pickle.load(f)
1365            if isinstance(data, dict):
1366                annotation = data
1367                for ind in annotation:
1368                    for cat, cat_list in annotation[ind].items():
1369                        annotation[ind][cat] = [
1370                            [start, end, 0] for start, end in cat_list
1371                        ]
1372            else:
1373                _, loaded_labels, animals, loaded_times = data
1374                annotation = {}
1375                for ind, ind_list in zip(animals, loaded_times):
1376                    annotation[ind] = {}
1377                    for cat, cat_list in zip(loaded_labels, ind_list):
1378                        annotation[ind][cat] = cat_list
1379            return annotation
1380        except:
1381            print(f"{filename} is invalid or does not exist")
1382            return None
1383
1384
1385class BorisAnnotationStore(FileAnnotationStore):  # +
1386    """
1387    BORIS type annotation data
1388
1389    Assumes the following file structure:
1390    ```
1391    annotation_path
1392    ├── video1_annotation.pickle
1393    └── video2_labels.pickle
1394    ```
1395    Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`.
1396    """
1397
1398    def __init__(
1399        self,
1400        video_order: List = None,
1401        min_frames: Dict = None,
1402        max_frames: Dict = None,
1403        visibility: Dict = None,
1404        exclusive: bool = True,
1405        len_segment: int = 128,
1406        overlap: int = 0,
1407        behaviors: Set = None,
1408        ignored_classes: Set = None,
1409        annotation_suffix: Union[Set, str] = None,
1410        annotation_path: Union[Set, str] = None,
1411        behavior_file: str = None,
1412        correction: Dict = None,
1413        frame_limit: int = 0,
1414        filter_annotated: bool = False,
1415        filter_background: bool = False,
1416        error_class: str = None,
1417        min_frames_action: int = None,
1418        key_objects: Tuple = None,
1419        visibility_min_score: float = 0.2,
1420        visibility_min_frac: float = 0.7,
1421        mask: Dict = None,
1422        use_hard_negatives: bool = False,
1423        default_agent_name: str = "ind0",
1424        interactive: bool = False,
1425        ignored_clips: Set = None,
1426        *args,
1427        **kwargs,
1428    ) -> None:
1429        """
1430        Parameters
1431        ----------
1432        video_order : list, optional
1433            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1434        min_frames : dict, optional
1435            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1436            clip start frames (not passed if creating from key objects)
1437        max_frames : dict, optional
1438            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1439            clip end frames (not passed if creating from key objects)
1440        visibility : dict, optional
1441            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1442            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
1443        exclusive : bool, default True
1444            if True, the annotation is single-label; if False, multi-label
1445        len_segment : int, default 128
1446            the length of the segments in which the data should be cut (in frames)
1447        overlap : int, default 0
1448            the length of the overlap between neighboring segments (in frames)
1449        behaviors : set, optional
1450            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
1451            loaded from a file)
1452        ignored_classes : set, optional
1453            the list of behaviors from the behaviors list or file to not annotate
1454        annotation_suffix : str | set, optional
1455            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
1456            (not passed if creating from key objects or if irrelevant for the dataset)
1457        annotation_path : str | set, optional
1458            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1459            from key objects)
1460        behavior_file : str, optional
1461            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
1462        correction : dict, optional
1463            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
1464            can be used to correct for variations in naming or to merge several labels in one
1465        frame_limit : int, default 0
1466            the smallest possible length of a clip (shorter clips are discarded)
1467        filter_annotated : bool, default False
1468            if True, the samples that do not have any labels will be filtered
1469        filter_background : bool, default False
1470            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
1471        error_class : str, optional
1472            the name of the error class (the annotations that intersect with this label will be discarded)
1473        min_frames_action : int, default 0
1474            the minimum length of an action (shorter actions are not annotated)
1475        key_objects : tuple, optional
1476            the key objects to load the AnnotationStore from
1477        visibility_min_score : float, default 5
1478            the minimum visibility score for visibility filtering
1479        visibility_min_frac : float, default 0.7
1480            the minimum fraction of visible frames for visibility filtering
1481        mask : dict, optional
1482            a masked value dictionary (for active learning simulation experiments)
1483        use_hard_negatives : bool, default False
1484            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
1485        interactive : bool, default False
1486            if `True`, annotation is assigned to pairs of individuals
1487        ignored_clips : set, optional
1488            a set of clip ids to ignore
1489        """
1490
1491        self.default_agent_name = default_agent_name
1492        super().__init__(
1493            video_order=video_order,
1494            min_frames=min_frames,
1495            max_frames=max_frames,
1496            visibility=visibility,
1497            exclusive=exclusive,
1498            len_segment=len_segment,
1499            overlap=overlap,
1500            behaviors=behaviors,
1501            ignored_classes=ignored_classes,
1502            annotation_suffix=annotation_suffix,
1503            annotation_path=annotation_path,
1504            behavior_file=behavior_file,
1505            correction=correction,
1506            frame_limit=frame_limit,
1507            filter_annotated=filter_annotated,
1508            filter_background=filter_background,
1509            error_class=error_class,
1510            min_frames_action=min_frames_action,
1511            key_objects=key_objects,
1512            visibility_min_score=visibility_min_score,
1513            visibility_min_frac=visibility_min_frac,
1514            mask=mask,
1515            use_hard_negatives=use_hard_negatives,
1516            interactive=interactive,
1517            ignored_clips=ignored_clips,
1518        )
1519
1520    def _open_annotations(self, filename: str) -> Dict:
1521        """
1522        Load the annotation from filename
1523        """
1524
1525        try:
1526            df = pd.read_csv(filename, header=15)
1527            fps = df.iloc[0]["FPS"]
1528            df["Subject"] = df["Subject"].fillna(self.default_agent_name)
1529            loaded_labels = list(df["Behavior"].unique())
1530            animals = list(df["Subject"].unique())
1531            loaded_times = {}
1532            for ind in animals:
1533                loaded_times[ind] = {}
1534                agent_df = df[df["Subject"] == ind]
1535                for cat in loaded_labels:
1536                    filtered_df = agent_df[agent_df["Behavior"] == cat]
1537                    starts = (
1538                        filtered_df["Time"][filtered_df["Status"] == "START"] * fps
1539                    ).astype(int)
1540                    ends = (
1541                        filtered_df["Time"][filtered_df["Status"] == "STOP"] * fps
1542                    ).astype(int)
1543                    loaded_times[ind][cat] = [
1544                        [start, end, 0] for start, end in zip(starts, ends)
1545                    ]
1546            return loaded_times
1547        except:
1548            print(f"{filename} is invalid or does not exist")
1549            return None
1550
1551
1552class PKUMMDAnnotationStore(FileAnnotationStore):  # +
1553    """
1554    PKU-MMD annotation data
1555
1556    Assumes the following file structure:
1557    ```
1558    annotation_path
1559    ├── 0364-L.txt
1560    ...
1561    └── 0144-M.txt
1562    ```
1563    """
1564
1565    def __init__(
1566        self,
1567        video_order: List = None,
1568        min_frames: Dict = None,
1569        max_frames: Dict = None,
1570        visibility: Dict = None,
1571        exclusive: bool = True,
1572        len_segment: int = 128,
1573        overlap: int = 0,
1574        ignored_classes: Set = None,
1575        annotation_path: Union[Set, str] = None,
1576        behavior_file: str = None,
1577        correction: Dict = None,
1578        frame_limit: int = 0,
1579        filter_annotated: bool = False,
1580        filter_background: bool = False,
1581        error_class: str = None,
1582        min_frames_action: int = None,
1583        key_objects: Tuple = None,
1584        visibility_min_score: float = 0,
1585        visibility_min_frac: float = 0,
1586        mask: Dict = None,
1587        use_hard_negatives: bool = False,
1588        interactive: bool = False,
1589        ignored_clips: Set = None,
1590        *args,
1591        **kwargs,
1592    ) -> None:
1593        """
1594        Parameters
1595        ----------
1596        video_order : list, optional
1597            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1598        min_frames : dict, optional
1599            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1600            clip start frames (not passed if creating from key objects)
1601        max_frames : dict, optional
1602            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1603            clip end frames (not passed if creating from key objects)
1604        visibility : dict, optional
1605            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1606            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
1607        exclusive : bool, default True
1608            if True, the annotation is single-label; if False, multi-label
1609        len_segment : int, default 128
1610            the length of the segments in which the data should be cut (in frames)
1611        overlap : int, default 0
1612            the length of the overlap between neighboring segments (in frames)
1613        ignored_classes : set, optional
1614            the list of behaviors from the behaviors list or file to not annotate
1615        annotation_path : str | set, optional
1616            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1617            from key objects)
1618        behavior_file : str, optional
1619            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
1620        correction : dict, optional
1621            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
1622            can be used to correct for variations in naming or to merge several labels in one
1623        frame_limit : int, default 0
1624            the smallest possible length of a clip (shorter clips are discarded)
1625        filter_annotated : bool, default False
1626            if True, the samples that do not have any labels will be filtered
1627        filter_background : bool, default False
1628            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
1629        error_class : str, optional
1630            the name of the error class (the annotations that intersect with this label will be discarded)
1631        min_frames_action : int, default 0
1632            the minimum length of an action (shorter actions are not annotated)
1633        key_objects : tuple, optional
1634            the key objects to load the AnnotationStore from
1635        visibility_min_score : float, default 5
1636            the minimum visibility score for visibility filtering
1637        visibility_min_frac : float, default 0.7
1638            the minimum fraction of visible frames for visibility filtering
1639        mask : dict, optional
1640            a masked value dictionary (for active learning simulation experiments)
1641        use_hard_negatives : bool, default False
1642            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
1643        interactive : bool, default False
1644            if `True`, annotation is assigned to pairs of individuals
1645        ignored_clips : set, optional
1646            a set of clip ids to ignore
1647        """
1648
1649        super().__init__(
1650            video_order=video_order,
1651            min_frames=min_frames,
1652            max_frames=max_frames,
1653            visibility=visibility,
1654            exclusive=exclusive,
1655            len_segment=len_segment,
1656            overlap=overlap,
1657            behaviors=None,
1658            ignored_classes=ignored_classes,
1659            annotation_suffix={".txt"},
1660            annotation_path=annotation_path,
1661            behavior_file=behavior_file,
1662            correction=correction,
1663            frame_limit=frame_limit,
1664            filter_annotated=filter_annotated,
1665            filter_background=filter_background,
1666            error_class=error_class,
1667            min_frames_action=min_frames_action,
1668            key_objects=key_objects,
1669            visibility_min_score=visibility_min_score,
1670            visibility_min_frac=visibility_min_frac,
1671            mask=mask,
1672            use_hard_negatives=use_hard_negatives,
1673            interactive=interactive,
1674            ignored_clips=ignored_clips,
1675            *args,
1676            **kwargs,
1677        )
1678
1679    @classmethod
1680    def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List:
1681        """
1682        Process data parameters and return a list of ids  of the videos that should
1683        be processed by the __init__ function
1684
1685        Parameters
1686        ----------
1687        annotation_path : str | set
1688            the path or the set of paths to the folder where the annotation files are stored
1689
1690        Returns
1691        -------
1692        video_ids : list
1693            a list of video file ids
1694        """
1695
1696        if isinstance(annotation_path, str):
1697            annotation_path = [annotation_path]
1698        files = []
1699        for folder in annotation_path:
1700            files += [
1701                os.path.basename(x)[:-4] for x in os.listdir(folder) if x[-4:] == ".txt"
1702            ]
1703        files = sorted(files, key=lambda x: os.path.basename(x))
1704        return files
1705
1706    def _open_annotations(self, filename: str) -> Dict:
1707        """
1708        Load the annotation from filename
1709        """
1710
1711        if self.interactive:
1712            agent_name = "0+1"
1713        else:
1714            agent_name = "0"
1715        times = {agent_name: defaultdict(lambda: [])}
1716        with open(filename) as f:
1717            for line in f.readlines():
1718                label, start, end, *_ = map(int, line.split(","))
1719                times[agent_name][self.all_behaviors[int(label) - 1]].append(
1720                    [start, end, 0]
1721                )
1722        return times
1723
1724    def _set_behaviors(
1725        self, behaviors: List, ignored_classes: List, behavior_file: str
1726    ):
1727        """
1728        Get a list of behaviors that should be annotated from behavior parameters
1729        """
1730
1731        if behavior_file is not None:
1732            behaviors = list(pd.read_excel(behavior_file)["Action"])
1733        self.all_behaviors = copy(behaviors)
1734        for b in ignored_classes:
1735            if b in behaviors:
1736                behaviors.remove(b)
1737        self.behaviors = behaviors
1738
1739
1740class CalMS21AnnotationStore(SequenceAnnotationStore):  # +
1741    """
1742    CalMS21 annotation data
1743
1744    Use the `'random:test_from_name:{name}'` and `'val-from-name:{val_name}:test-from-name:{test_name}'`
1745    partitioning methods with `'train'`, `'test'` and `'unlabeled'` names to separate into train, test and validation
1746    subsets according to the original files. For example, with `'val-from-name:test:test-from-name:unlabeled'`
1747    the data from the test file will go into validation and the unlabeled files will be the test.
1748
1749    Assumes the following file structure:
1750    ```
1751    annotation_path
1752    ├── calms21_task_train.npy
1753    ├── calms21_task_test.npy
1754    ├── calms21_unlabeled_videos_part1.npy
1755    ├── calms21_unlabeled_videos_part2.npy
1756    └── calms21_unlabeled_videos_part3.npy
1757    ```
1758    """
1759
1760    def __init__(
1761        self,
1762        task_n: int = 1,
1763        include_task1: bool = True,
1764        video_order: List = None,
1765        min_frames: Dict = None,
1766        max_frames: Dict = None,
1767        len_segment: int = 128,
1768        overlap: int = 0,
1769        ignored_classes: Set = None,
1770        annotation_path: Union[Set, str] = None,
1771        key_objects: Tuple = None,
1772        treba_files: bool = False,
1773        *args,
1774        **kwargs,
1775    ) -> None:
1776        """
1777
1778        Parameters
1779        ----------
1780        task_n : [1, 2]
1781            the number of the task
1782        include_task1 : bool, default True
1783            include task 1 data to training set
1784        video_order : list, optional
1785            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1786        min_frames : dict, optional
1787            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1788            clip start frames (not passed if creating from key objects)
1789        max_frames : dict, optional
1790            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1791            clip end frames (not passed if creating from key objects)
1792        len_segment : int, default 128
1793            the length of the segments in which the data should be cut (in frames)
1794        overlap : int, default 0
1795            the length of the overlap between neighboring segments (in frames)
1796        ignored_classes : set, optional
1797            the list of behaviors from the behaviors list or file to not annotate
1798        annotation_path : str | set, optional
1799            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1800            from key objects)
1801        key_objects : tuple, optional
1802            the key objects to load the AnnotationStore from
1803        treba_files : bool, default False
1804            if `True`, TREBA feature files will be loaded
1805        """
1806
1807        self.task_n = int(task_n)
1808        self.include_task1 = include_task1
1809        if self.task_n == 1:
1810            self.include_task1 = False
1811        self.treba_files = treba_files
1812        if "exclusive" in kwargs:
1813            exclusive = kwargs["exclusive"]
1814        else:
1815            exclusive = True
1816        if "behaviors" in kwargs and kwargs["behaviors"] is not None:
1817            behaviors = kwargs["behaviors"]
1818        else:
1819            behaviors = ["attack", "investigation", "mount", "other"]
1820            if task_n == 3:
1821                exclusive = False
1822                behaviors += [
1823                    "approach",
1824                    "disengaged",
1825                    "groom",
1826                    "intromission",
1827                    "mount_attempt",
1828                    "sniff_face",
1829                    "whiterearing",
1830                ]
1831        super().__init__(
1832            video_order=video_order,
1833            min_frames=min_frames,
1834            max_frames=max_frames,
1835            exclusive=exclusive,
1836            len_segment=len_segment,
1837            overlap=overlap,
1838            behaviors=behaviors,
1839            ignored_classes=ignored_classes,
1840            annotation_path=annotation_path,
1841            key_objects=key_objects,
1842            filter_annotated=False,
1843            interactive=True,
1844        )
1845
1846    @classmethod
1847    def get_file_ids(
1848        cls,
1849        task_n: int = 1,
1850        include_task1: bool = False,
1851        treba_files: bool = False,
1852        annotation_path: Union[str, Set] = None,
1853        file_paths=None,
1854        *args,
1855        **kwargs,
1856    ) -> Iterable:
1857        """
1858        Process data parameters and return a list of ids  of the videos that should
1859        be processed by the __init__ function
1860
1861        Parameters
1862        ----------
1863        task_n : {1, 2, 3}
1864            the index of the CalMS21 challenge task
1865        include_task1 : bool, default False
1866            if `True`, the training file of the task 1 will be loaded
1867        treba_files : bool, default False
1868            if `True`, the TREBA feature files will be loaded
1869        filenames : set, optional
1870            a set of string filenames to search for (only basenames, not the whole paths)
1871        annotation_path : str | set, optional
1872            the path to the folder where the pose and feature files are stored or a set of such paths
1873            (not passed if creating from key objects or from `file_paths`)
1874        file_paths : set, optional
1875            a set of string paths to the pose and feature files
1876            (not passed if creating from key objects or from `data_path`)
1877
1878        Returns
1879        -------
1880        video_ids : list
1881            a list of video file ids
1882        """
1883
1884        task_n = int(task_n)
1885        if task_n == 1:
1886            include_task1 = False
1887        files = []
1888        if treba_files:
1889            postfix = "_features"
1890        else:
1891            postfix = ""
1892        files.append(f"calms21_task{task_n}_train{postfix}.npy")
1893        files.append(f"calms21_task{task_n}_test{postfix}.npy")
1894        if include_task1:
1895            files.append(f"calms21_task1_train{postfix}.npy")
1896        filenames = set(files)
1897        return SequenceAnnotationStore.get_file_ids(
1898            filenames, annotation_path=annotation_path
1899        )
1900
1901    def _open_sequences(self, filename: str) -> Dict:
1902        """
1903        Load the annotation from filename
1904
1905        Parameters
1906        ----------
1907        filename : str
1908            path to an annotation file
1909
1910        Returns
1911        -------
1912        times : dict
1913            a nested dictionary where first-level keys are video ids, second-level keys are clip ids,
1914            third-level keys are categories and values are
1915            lists of (start, end, ambiguity status) lists
1916        """
1917
1918        data_dict = np.load(filename, allow_pickle=True).item()
1919        data = {}
1920        result = {}
1921        keys = list(data_dict.keys())
1922        if "test" in os.path.basename(filename):
1923            mode = "test"
1924        elif "unlabeled" in os.path.basename(filename):
1925            mode = "unlabeled"
1926        else:
1927            mode = "train"
1928        if "approach" in keys:
1929            for behavior in keys:
1930                for key in data_dict[behavior].keys():
1931                    ann = data_dict[behavior][key]["annotations"]
1932                    result[f'{mode}--{key.split("/")[-1]}'] = {
1933                        "mouse1+mouse2": defaultdict(lambda: [])
1934                    }
1935                    starts = np.where(
1936                        np.diff(np.concatenate([np.array([0]), ann, np.array([0])]))
1937                        == 1
1938                    )[0]
1939                    ends = np.where(
1940                        np.diff(np.concatenate([np.array([0]), ann, np.array([0])]))
1941                        == -1
1942                    )[0]
1943                    for start, end in zip(starts, ends):
1944                        result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][
1945                            behavior
1946                        ].append([start, end, 0])
1947                    for b in self.behaviors:
1948                        result[f'{mode}--{key.split("/")[-1]}---mouse1+mouse2'][
1949                            "mouse1+mouse2"
1950                        ][f"unknown {b}"].append([0, len(ann), 0])
1951        for key in keys:
1952            data.update(data_dict[key])
1953            data_dict.pop(key)
1954        if "approach" not in keys and self.task_n == 3:
1955            for key in data.keys():
1956                result[f'{mode}--{key.split("/")[-1]}'] = {"mouse1+mouse2": {}}
1957                ann = data[key]["annotations"]
1958                for i in range(4):
1959                    starts = np.where(
1960                        np.diff(
1961                            np.concatenate(
1962                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1963                            )
1964                        )
1965                        == 1
1966                    )[0]
1967                    ends = np.where(
1968                        np.diff(
1969                            np.concatenate(
1970                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1971                            )
1972                        )
1973                        == -1
1974                    )[0]
1975                    result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][
1976                        self.behaviors_dict()[i]
1977                    ] = [[start, end, 0] for start, end in zip(starts, ends)]
1978        if self.task_n != 3:
1979            for seq_name, seq_dict in data.items():
1980                if "annotations" not in seq_dict:
1981                    return None
1982                behaviors = np.unique(seq_dict["annotations"])
1983                ann = seq_dict["annotations"]
1984                key = f'{mode}--{seq_name.split("/")[-1]}'
1985                result[key] = {"mouse1+mouse2": {}}
1986                for i in behaviors:
1987                    starts = np.where(
1988                        np.diff(
1989                            np.concatenate(
1990                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1991                            )
1992                        )
1993                        == 1
1994                    )[0]
1995                    ends = np.where(
1996                        np.diff(
1997                            np.concatenate(
1998                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1999                            )
2000                        )
2001                        == -1
2002                    )[0]
2003                    result[key]["mouse1+mouse2"][self.behaviors_dict()[i]] = [
2004                        [start, end, 0] for start, end in zip(starts, ends)
2005                    ]
2006        return result
2007
2008
2009class CSVAnnotationStore(FileAnnotationStore):  # +
2010    """
2011    CSV type annotation data
2012
2013    Assumes that files are saved as .csv tables with at least the following columns:
2014    - from / start : start of action,
2015    - to / end : end of action,
2016    - class / behavior / behaviour / label / type : action label.
2017
2018    If the times are set in seconds instead of frames, don't forget to set the `fps` parameter to your frame rate.
2019
2020    Assumes the following file structure:
2021    ```
2022    annotation_path
2023    ├── video1_annotation.csv
2024    └── video2_labels.csv
2025    ```
2026    Here `annotation_suffix` is `{'_annotation.csv', '_labels.csv'}`.
2027    """
2028
2029    def __init__(
2030        self,
2031        video_order: List = None,
2032        min_frames: Dict = None,
2033        max_frames: Dict = None,
2034        visibility: Dict = None,
2035        exclusive: bool = True,
2036        len_segment: int = 128,
2037        overlap: int = 0,
2038        behaviors: Set = None,
2039        ignored_classes: Set = None,
2040        annotation_suffix: Union[Set, str] = None,
2041        annotation_path: Union[Set, str] = None,
2042        behavior_file: str = None,
2043        correction: Dict = None,
2044        frame_limit: int = 0,
2045        filter_annotated: bool = False,
2046        filter_background: bool = False,
2047        error_class: str = None,
2048        min_frames_action: int = None,
2049        key_objects: Tuple = None,
2050        visibility_min_score: float = 0.2,
2051        visibility_min_frac: float = 0.7,
2052        mask: Dict = None,
2053        default_agent_name: str = "ind0",
2054        separator: str = ",",
2055        fps: int = 30,
2056        *args,
2057        **kwargs,
2058    ) -> None:
2059        """
2060        Parameters
2061        ----------
2062        video_order : list, optional
2063            a list of video ids that should be processed in the same order (not passed if creating from key objects)
2064        min_frames : dict, optional
2065            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2066            clip start frames (not passed if creating from key objects)
2067        max_frames : dict, optional
2068            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2069            clip end frames (not passed if creating from key objects)
2070        visibility : dict, optional
2071            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2072            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
2073        exclusive : bool, default True
2074            if True, the annotation is single-label; if False, multi-label
2075        len_segment : int, default 128
2076            the length of the segments in which the data should be cut (in frames)
2077        overlap : int, default 0
2078            the length of the overlap between neighboring segments (in frames)
2079        behaviors : set, optional
2080            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
2081            loaded from a file)
2082        ignored_classes : set, optional
2083            the list of behaviors from the behaviors list or file to not annotate
2084        annotation_suffix : str | set, optional
2085            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
2086            (not passed if creating from key objects or if irrelevant for the dataset)
2087        annotation_path : str | set, optional
2088            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
2089            from key objects)
2090        behavior_file : str, optional
2091            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
2092        correction : dict, optional
2093            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
2094            can be used to correct for variations in naming or to merge several labels in one
2095        frame_limit : int, default 0
2096            the smallest possible length of a clip (shorter clips are discarded)
2097        filter_annotated : bool, default False
2098            if True, the samples that do not have any labels will be filtered
2099        filter_background : bool, default False
2100            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
2101        error_class : str, optional
2102            the name of the error class (the annotations that intersect with this label will be discarded)
2103        min_frames_action : int, default 0
2104            the minimum length of an action (shorter actions are not annotated)
2105        key_objects : tuple, optional
2106            the key objects to load the AnnotationStore from
2107        visibility_min_score : float, default 5
2108            the minimum visibility score for visibility filtering
2109        visibility_min_frac : float, default 0.7
2110            the minimum fraction of visible frames for visibility filtering
2111        mask : dict, optional
2112            a masked value dictionary (for active learning simulation experiments)
2113        default_agent_name : str, default "ind0"
2114            the clip id to use when there is no given
2115        separator : str, default ","
2116            the separator in the csv files
2117        fps : int, default 30
2118            frames per second in the videos
2119        """
2120
2121        self.default_agent_name = default_agent_name
2122        self.separator = separator
2123        self.fps = fps
2124        super().__init__(
2125            video_order=video_order,
2126            min_frames=min_frames,
2127            max_frames=max_frames,
2128            visibility=visibility,
2129            exclusive=exclusive,
2130            len_segment=len_segment,
2131            overlap=overlap,
2132            behaviors=behaviors,
2133            ignored_classes=ignored_classes,
2134            ignored_clips=None,
2135            annotation_suffix=annotation_suffix,
2136            annotation_path=annotation_path,
2137            behavior_file=behavior_file,
2138            correction=correction,
2139            frame_limit=frame_limit,
2140            filter_annotated=filter_annotated,
2141            filter_background=filter_background,
2142            error_class=error_class,
2143            min_frames_action=min_frames_action,
2144            key_objects=key_objects,
2145            visibility_min_score=visibility_min_score,
2146            visibility_min_frac=visibility_min_frac,
2147            mask=mask,
2148        )
2149
2150    def _open_annotations(self, filename: str) -> Dict:
2151        """
2152        Load the annotation from `filename`
2153        """
2154
2155        # try:
2156        data = pd.read_csv(filename, sep=self.separator)
2157        data.columns = list(map(lambda x: x.lower(), data.columns))
2158        starts, ends, actions = None, None, None
2159        start_names = ["from", "start"]
2160        for x in start_names:
2161            if x in data.columns:
2162                starts = data[x]
2163        end_names = ["to", "end"]
2164        for x in end_names:
2165            if x in data.columns:
2166                ends = data[x]
2167        class_names = ["class", "behavior", "behaviour", "type", "label"]
2168        for x in class_names:
2169            if x in data.columns:
2170                actions = data[x]
2171        if starts is None:
2172            raise ValueError("The file must have a column titled 'from' or 'start'!")
2173        if ends is None:
2174            raise ValueError("The file must have a column titled 'to' or 'end'!")
2175        if actions is None:
2176            raise ValueError(
2177                "The file must have a column titled 'class', 'behavior', 'behaviour', 'type' or 'label'!"
2178            )
2179        times = defaultdict(lambda: defaultdict(lambda: []))
2180        for start, end, action in zip(starts, ends, actions):
2181            if any([np.isnan(x) for x in [start, end]]):
2182                continue
2183            times[self.default_agent_name][action].append(
2184                [int(start * self.fps), int(end * self.fps), 0]
2185            )
2186        return times
2187
2188
2189class SIMBAAnnotationStore(FileAnnotationStore):  # +
2190    """
2191    SIMBA paper format data
2192
2193    Assumes the following file structure:
2194    ```
2195    annotation_path
2196    ├── Video1.csv
2197    ...
2198    └── Video9.csv
2199    """
2200
2201    def __init__(
2202        self,
2203        video_order: List = None,
2204        min_frames: Dict = None,
2205        max_frames: Dict = None,
2206        visibility: Dict = None,
2207        exclusive: bool = True,
2208        len_segment: int = 128,
2209        overlap: int = 0,
2210        behaviors: Set = None,
2211        ignored_classes: Set = None,
2212        ignored_clips: Set = None,
2213        annotation_path: Union[Set, str] = None,
2214        correction: Dict = None,
2215        filter_annotated: bool = False,
2216        filter_background: bool = False,
2217        error_class: str = None,
2218        min_frames_action: int = None,
2219        key_objects: Tuple = None,
2220        visibility_min_score: float = 0.2,
2221        visibility_min_frac: float = 0.7,
2222        mask: Dict = None,
2223        use_hard_negatives: bool = False,
2224        annotation_suffix: str = None,
2225        *args,
2226        **kwargs,
2227    ) -> None:
2228        """
2229        Parameters
2230        ----------
2231        video_order : list, optional
2232            a list of video ids that should be processed in the same order (not passed if creating from key objects)
2233        min_frames : dict, optional
2234            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2235            clip start frames (not passed if creating from key objects)
2236        max_frames : dict, optional
2237            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2238            clip end frames (not passed if creating from key objects)
2239        visibility : dict, optional
2240            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2241            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
2242        exclusive : bool, default True
2243            if True, the annotation is single-label; if False, multi-label
2244        len_segment : int, default 128
2245            the length of the segments in which the data should be cut (in frames)
2246        overlap : int, default 0
2247            the length of the overlap between neighboring segments (in frames)
2248        behaviors : set, optional
2249            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
2250            loaded from a file)
2251        ignored_classes : set, optional
2252            the list of behaviors from the behaviors list or file to not annotate
2253        ignored_clips : set, optional
2254            clip ids to ignore
2255        annotation_path : str | set, optional
2256            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
2257            from key objects)
2258        behavior_file : str, optional
2259            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
2260        correction : dict, optional
2261            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
2262            can be used to correct for variations in naming or to merge several labels in one
2263        filter_annotated : bool, default False
2264            if True, the samples that do not have any labels will be filtered
2265        filter_background : bool, default False
2266            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
2267        error_class : str, optional
2268            the name of the error class (the annotations that intersect with this label will be discarded)
2269        min_frames_action : int, default 0
2270            the minimum length of an action (shorter actions are not annotated)
2271        key_objects : tuple, optional
2272            the key objects to load the AnnotationStore from
2273        visibility_min_score : float, default 5
2274            the minimum visibility score for visibility filtering
2275        visibility_min_frac : float, default 0.7
2276            the minimum fraction of visible frames for visibility filtering
2277        mask : dict, optional
2278            a masked value dictionary (for active learning simulation experiments)
2279        use_hard_negatives : bool, default False
2280            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
2281        annotation_suffix : str | set, optional
2282            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
2283            (not passed if creating from key objects or if irrelevant for the dataset)
2284        """
2285
2286        super().__init__(
2287            video_order=video_order,
2288            min_frames=min_frames,
2289            max_frames=max_frames,
2290            visibility=visibility,
2291            exclusive=exclusive,
2292            len_segment=len_segment,
2293            overlap=overlap,
2294            behaviors=behaviors,
2295            ignored_classes=ignored_classes,
2296            ignored_clips=ignored_clips,
2297            annotation_suffix=annotation_suffix,
2298            annotation_path=annotation_path,
2299            behavior_file=None,
2300            correction=correction,
2301            frame_limit=0,
2302            filter_annotated=filter_annotated,
2303            filter_background=filter_background,
2304            error_class=error_class,
2305            min_frames_action=min_frames_action,
2306            key_objects=key_objects,
2307            visibility_min_score=visibility_min_score,
2308            visibility_min_frac=visibility_min_frac,
2309            mask=mask,
2310            use_hard_negatives=use_hard_negatives,
2311            interactive=True,
2312        )
2313
2314    def _open_annotations(self, filename: str) -> Dict:
2315        """
2316        Load the annotation from filename
2317
2318        Parameters
2319        ----------
2320        filename : str
2321            path to an annotation file
2322
2323        Returns
2324        -------
2325        times : dict
2326            a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are
2327            lists of (start, end, ambiguity status) lists
2328        """
2329
2330        data = pd.read_csv(filename)
2331        columns = [x for x in data.columns if x.split("_")[-1] == "x"]
2332        animals = sorted(set([x.split("_")[-2] for x in columns]))
2333        if len(animals) > 2:
2334            raise ValueError(
2335                "SIMBAAnnotationStore is only implemented for files with 1 or 2 animals!"
2336            )
2337        if len(animals) == 1:
2338            ind = animals[0]
2339        else:
2340            ind = "+".join(animals)
2341        behaviors = [
2342            "_".join(x.split("_")[:-1])
2343            for x in data.columns
2344            if x.split("_")[-1] == "prediction"
2345        ]
2346        result = {}
2347        for behavior in behaviors:
2348            ann = data[f"{behavior}_prediction"].values
2349            diff = np.diff(
2350                np.concatenate([np.array([0]), (ann == 1).astype(int), np.array([0])])
2351            )
2352            starts = np.where(diff == 1)[0]
2353            ends = np.where(diff == -1)[0]
2354            result[behavior] = [[start, end, 0] for start, end in zip(starts, ends)]
2355            diff = np.diff(
2356                np.concatenate(
2357                    [np.array([0]), (np.isnan(ann)).astype(int), np.array([0])]
2358                )
2359            )
2360            starts = np.where(diff == 1)[0]
2361            ends = np.where(diff == -1)[0]
2362            result[f"unknown {behavior}"] = [
2363                [start, end, 0] for start, end in zip(starts, ends)
2364            ]
2365        if self.behaviors is not None:
2366            for behavior in self.behaviors:
2367                if behavior not in behaviors:
2368                    result[f"unknown {behavior}"] = [[0, len(data), 0]]
2369        return {ind: result}
class EmptyAnnotationStore(dlc2action.data.base_store.AnnotationStore):
 28class EmptyAnnotationStore(AnnotationStore):
 29    def __init__(
 30        self, video_order: List = None, key_objects: Tuple = None, *args, **kwargs
 31    ):
 32        """
 33        Parameters
 34        ----------
 35        video_order : list, optional
 36            a list of video ids that should be processed in the same order (not passed if creating from key objects)
 37        key_objects : tuple, optional
 38            a tuple of key objects
 39        """
 40
 41        pass
 42
 43    def __len__(self) -> int:
 44        """
 45        Get the number of available samples
 46
 47        Returns
 48        -------
 49        length : int
 50            the number of available samples
 51        """
 52
 53        return 0
 54
 55    def remove(self, indices: List) -> None:
 56        """
 57        Remove the samples corresponding to indices
 58
 59        Parameters
 60        ----------
 61        indices : int
 62            a list of integer indices to remove
 63        """
 64
 65        pass
 66
 67    def key_objects(self) -> Tuple:
 68        """
 69        Return a tuple of the key objects necessary to re-create the Store
 70
 71        Returns
 72        -------
 73        key_objects : tuple
 74            a tuple of key objects
 75        """
 76
 77        return ()
 78
 79    def load_from_key_objects(self, key_objects: Tuple) -> None:
 80        """
 81        Load the information from a tuple of key objects
 82
 83        Parameters
 84        ----------
 85        key_objects : tuple
 86            a tuple of key objects
 87        """
 88
 89        pass
 90
 91    def to_ram(self) -> None:
 92        """
 93        Transfer the data samples to RAM if they were previously stored as file paths
 94        """
 95
 96        pass
 97
 98    def get_original_coordinates(self) -> np.ndarray:
 99        """
100        Return the original coordinates array
101
102        Returns
103        -------
104        np.ndarray
105            an array that contains the coordinates of the data samples in original input data (video id, clip id,
106            start frame)
107        """
108
109        return None
110
111    def create_subsample(self, indices: List, ssl_indices: List = None):
112        """
113        Create a new store that contains a subsample of the data
114
115        Parameters
116        ----------
117        indices : list
118            the indices to be included in the subsample
119        ssl_indices : list, optional
120            the indices to be included in the subsample without the annotation data
121        """
122
123        return self.new()
124
125    @classmethod
126    def get_file_ids(cls, *args, **kwargs) -> List:
127        """
128        Process data parameters and return a list of ids  of the videos that should
129        be processed by the __init__ function
130
131        Returns
132        -------
133        video_ids : list
134            a list of video file ids
135        """
136
137        return None
138
139    def __getitem__(self, ind: int) -> torch.Tensor:
140        """
141        Return the annotation of the sample corresponding to an index
142
143        Parameters
144        ----------
145        ind : int
146            index of the sample
147
148        Returns
149        -------
150        sample : torch.Tensor
151            the corresponding annotation tensor
152        """
153
154        return torch.tensor(float("nan"))
155
156    def get_len(self, return_unlabeled: bool) -> int:
157        """
158        Get the length of the subsample of labeled/unlabeled data
159
160        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
161        and if return_unlabeled is None the index is already correct
162
163        Parameters
164        ----------
165        return_unlabeled : bool
166            the identifier for the subsample
167
168        Returns
169        -------
170        length : int
171            the length of the subsample
172        """
173
174        return None
175
176    def get_idx(self, index: int, return_unlabeled: bool) -> int:
177        """
178        Convert from an index in the subsample of labeled/unlabeled data to an index in the full array
179
180        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
181        and if return_unlabeled is None the index is already correct
182
183        Parameters
184        ----------
185        index : int
186            the index in the subsample
187        return_unlabeled : bool
188            the identifier for the subsample
189
190        Returns
191        -------
192        corrected_index : int
193            the index in the full dataset
194        """
195
196        return index
197
198    def count_classes(
199        self, frac: bool = False, zeros: bool = False, bouts: bool = False
200    ) -> Dict:
201        """
202        Get a dictionary with class-wise frame counts
203
204        Parameters
205        ----------
206        frac : bool, default False
207            if True, a fraction of the total frame count is returned
208
209        Returns
210        -------
211        count_dictionary : dict
212            a dictionary with class indices as keys and frame counts as values
213        """
214
215        return {}
216
217    def behaviors_dict(self) -> Dict:
218        """
219        Get a dictionary of class names
220
221        Returns
222        -------
223        behavior_dictionary: dict
224            a dictionary with class indices as keys and class names as values
225        """
226
227        return {}
228
229    def annotation_class(self) -> str:
230        """
231        Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon)
232
233        Returns
234        -------
235        annotation_class : str
236            the type of annotation
237        """
238
239        return "none"
240
241    def size(self) -> int:
242        """
243        Get the total number of frames in the data
244
245        Returns
246        -------
247        size : int
248            the total number of frames
249        """
250
251        return None
252
253    def filtered_indices(self) -> List:
254        """
255        Return the indices of the samples that should be removed
256
257        Choosing the indices can be based on any kind of filering defined in the __init__ function by the data
258        parameters
259
260        Returns
261        -------
262        indices_to_remove : list
263            a list of integer indices that should be removed
264        """
265
266        return []
267
268    def set_pseudo_labels(self, labels: torch.Tensor) -> None:
269        """
270        Set pseudo labels to the unlabeled data
271
272        Parameters
273        ----------
274        labels : torch.Tensor
275            a tensor of pseudo-labels for the unlabeled data
276        """
277
278        pass

A class that generates annotation from video information and stores it

Processes input video information and generates ordered arrays of annotation samples and corresponding unique original coordinates, as well as some meta objects. It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). The original coordinates object contains information about the video_id, clip_id and start time of the samples in the original input data. An AnnotationStore has to be fully defined with a tuple of key objects. The annotation array can be accessed with integer indices. The samples can be stored as a torch.Tensor in RAM or as an array of file paths to be loaded on runtime. When no arguments are passed a blank class instance should be created that can later be filled with information from key objects

EmptyAnnotationStore(video_order: List = None, key_objects: Tuple = None, *args, **kwargs)
29    def __init__(
30        self, video_order: List = None, key_objects: Tuple = None, *args, **kwargs
31    ):
32        """
33        Parameters
34        ----------
35        video_order : list, optional
36            a list of video ids that should be processed in the same order (not passed if creating from key objects)
37        key_objects : tuple, optional
38            a tuple of key objects
39        """
40
41        pass

Parameters

video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) key_objects : tuple, optional a tuple of key objects

def remove(self, indices: List) -> None:
55    def remove(self, indices: List) -> None:
56        """
57        Remove the samples corresponding to indices
58
59        Parameters
60        ----------
61        indices : int
62            a list of integer indices to remove
63        """
64
65        pass

Remove the samples corresponding to indices

Parameters

indices : int a list of integer indices to remove

def key_objects(self) -> Tuple:
67    def key_objects(self) -> Tuple:
68        """
69        Return a tuple of the key objects necessary to re-create the Store
70
71        Returns
72        -------
73        key_objects : tuple
74            a tuple of key objects
75        """
76
77        return ()

Return a tuple of the key objects necessary to re-create the Store

Returns

key_objects : tuple a tuple of key objects

def load_from_key_objects(self, key_objects: Tuple) -> None:
79    def load_from_key_objects(self, key_objects: Tuple) -> None:
80        """
81        Load the information from a tuple of key objects
82
83        Parameters
84        ----------
85        key_objects : tuple
86            a tuple of key objects
87        """
88
89        pass

Load the information from a tuple of key objects

Parameters

key_objects : tuple a tuple of key objects

def to_ram(self) -> None:
91    def to_ram(self) -> None:
92        """
93        Transfer the data samples to RAM if they were previously stored as file paths
94        """
95
96        pass

Transfer the data samples to RAM if they were previously stored as file paths

def get_original_coordinates(self) -> numpy.ndarray:
 98    def get_original_coordinates(self) -> np.ndarray:
 99        """
100        Return the original coordinates array
101
102        Returns
103        -------
104        np.ndarray
105            an array that contains the coordinates of the data samples in original input data (video id, clip id,
106            start frame)
107        """
108
109        return None

Return the original coordinates array

Returns

np.ndarray an array that contains the coordinates of the data samples in original input data (video id, clip id, start frame)

def create_subsample(self, indices: List, ssl_indices: List = None)
111    def create_subsample(self, indices: List, ssl_indices: List = None):
112        """
113        Create a new store that contains a subsample of the data
114
115        Parameters
116        ----------
117        indices : list
118            the indices to be included in the subsample
119        ssl_indices : list, optional
120            the indices to be included in the subsample without the annotation data
121        """
122
123        return self.new()

Create a new store that contains a subsample of the data

Parameters

indices : list the indices to be included in the subsample ssl_indices : list, optional the indices to be included in the subsample without the annotation data

@classmethod
def get_file_ids(cls, *args, **kwargs) -> List:
125    @classmethod
126    def get_file_ids(cls, *args, **kwargs) -> List:
127        """
128        Process data parameters and return a list of ids  of the videos that should
129        be processed by the __init__ function
130
131        Returns
132        -------
133        video_ids : list
134            a list of video file ids
135        """
136
137        return None

Process data parameters and return a list of ids of the videos that should be processed by the __init__ function

Returns

video_ids : list a list of video file ids

def get_len(self, return_unlabeled: bool) -> int:
156    def get_len(self, return_unlabeled: bool) -> int:
157        """
158        Get the length of the subsample of labeled/unlabeled data
159
160        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
161        and if return_unlabeled is None the index is already correct
162
163        Parameters
164        ----------
165        return_unlabeled : bool
166            the identifier for the subsample
167
168        Returns
169        -------
170        length : int
171            the length of the subsample
172        """
173
174        return None

Get the length of the subsample of labeled/unlabeled data

If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct

Parameters

return_unlabeled : bool the identifier for the subsample

Returns

length : int the length of the subsample

def get_idx(self, index: int, return_unlabeled: bool) -> int:
176    def get_idx(self, index: int, return_unlabeled: bool) -> int:
177        """
178        Convert from an index in the subsample of labeled/unlabeled data to an index in the full array
179
180        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
181        and if return_unlabeled is None the index is already correct
182
183        Parameters
184        ----------
185        index : int
186            the index in the subsample
187        return_unlabeled : bool
188            the identifier for the subsample
189
190        Returns
191        -------
192        corrected_index : int
193            the index in the full dataset
194        """
195
196        return index

Convert from an index in the subsample of labeled/unlabeled data to an index in the full array

If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct

Parameters

index : int the index in the subsample return_unlabeled : bool the identifier for the subsample

Returns

corrected_index : int the index in the full dataset

def count_classes( self, frac: bool = False, zeros: bool = False, bouts: bool = False) -> Dict:
198    def count_classes(
199        self, frac: bool = False, zeros: bool = False, bouts: bool = False
200    ) -> Dict:
201        """
202        Get a dictionary with class-wise frame counts
203
204        Parameters
205        ----------
206        frac : bool, default False
207            if True, a fraction of the total frame count is returned
208
209        Returns
210        -------
211        count_dictionary : dict
212            a dictionary with class indices as keys and frame counts as values
213        """
214
215        return {}

Get a dictionary with class-wise frame counts

Parameters

frac : bool, default False if True, a fraction of the total frame count is returned

Returns

count_dictionary : dict a dictionary with class indices as keys and frame counts as values

def behaviors_dict(self) -> Dict:
217    def behaviors_dict(self) -> Dict:
218        """
219        Get a dictionary of class names
220
221        Returns
222        -------
223        behavior_dictionary: dict
224            a dictionary with class indices as keys and class names as values
225        """
226
227        return {}

Get a dictionary of class names

Returns

behavior_dictionary: dict a dictionary with class indices as keys and class names as values

def annotation_class(self) -> str:
229    def annotation_class(self) -> str:
230        """
231        Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon)
232
233        Returns
234        -------
235        annotation_class : str
236            the type of annotation
237        """
238
239        return "none"

Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon)

Returns

annotation_class : str the type of annotation

def size(self) -> int:
241    def size(self) -> int:
242        """
243        Get the total number of frames in the data
244
245        Returns
246        -------
247        size : int
248            the total number of frames
249        """
250
251        return None

Get the total number of frames in the data

Returns

size : int the total number of frames

def filtered_indices(self) -> List:
253    def filtered_indices(self) -> List:
254        """
255        Return the indices of the samples that should be removed
256
257        Choosing the indices can be based on any kind of filering defined in the __init__ function by the data
258        parameters
259
260        Returns
261        -------
262        indices_to_remove : list
263            a list of integer indices that should be removed
264        """
265
266        return []

Return the indices of the samples that should be removed

Choosing the indices can be based on any kind of filering defined in the __init__ function by the data parameters

Returns

indices_to_remove : list a list of integer indices that should be removed

def set_pseudo_labels(self, labels: torch.Tensor) -> None:
268    def set_pseudo_labels(self, labels: torch.Tensor) -> None:
269        """
270        Set pseudo labels to the unlabeled data
271
272        Parameters
273        ----------
274        labels : torch.Tensor
275            a tensor of pseudo-labels for the unlabeled data
276        """
277
278        pass

Set pseudo labels to the unlabeled data

Parameters

labels : torch.Tensor a tensor of pseudo-labels for the unlabeled data

class ActionSegmentationStore(dlc2action.data.base_store.AnnotationStore):
 281class ActionSegmentationStore(AnnotationStore):  # +
 282    """
 283    A general realization of an annotation store for action segmentation tasks
 284
 285    Assumes the following file structure:
 286    ```
 287    annotation_path
 288    ├── video1_annotation.pickle
 289    └── video2_labels.pickle
 290    ```
 291    Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`.
 292    """
 293
 294    def __init__(
 295        self,
 296        video_order: List = None,
 297        min_frames: Dict = None,
 298        max_frames: Dict = None,
 299        visibility: Dict = None,
 300        exclusive: bool = True,
 301        len_segment: int = 128,
 302        overlap: int = 0,
 303        behaviors: Set = None,
 304        ignored_classes: Set = None,
 305        ignored_clips: Set = None,
 306        annotation_suffix: Union[Set, str] = None,
 307        annotation_path: Union[Set, str] = None,
 308        behavior_file: str = None,
 309        correction: Dict = None,
 310        frame_limit: int = 0,
 311        filter_annotated: bool = False,
 312        filter_background: bool = False,
 313        error_class: str = None,
 314        min_frames_action: int = None,
 315        key_objects: Tuple = None,
 316        visibility_min_score: float = 0.2,
 317        visibility_min_frac: float = 0.7,
 318        mask: Dict = None,
 319        use_hard_negatives: bool = False,
 320        interactive: bool = False,
 321        *args,
 322        **kwargs,
 323    ) -> None:
 324        """
 325        Parameters
 326        ----------
 327        video_order : list, optional
 328            a list of video ids that should be processed in the same order (not passed if creating from key objects)
 329        min_frames : dict, optional
 330            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
 331            clip start frames (not passed if creating from key objects)
 332        max_frames : dict, optional
 333            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
 334            clip end frames (not passed if creating from key objects)
 335        visibility : dict, optional
 336            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
 337            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
 338        exclusive : bool, default True
 339            if True, the annotation is single-label; if False, multi-label
 340        len_segment : int, default 128
 341            the length of the segments in which the data should be cut (in frames)
 342        overlap : int, default 0
 343            the length of the overlap between neighboring segments (in frames)
 344        behaviors : set, optional
 345            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
 346            loaded from a file)
 347        ignored_classes : set, optional
 348            the list of behaviors from the behaviors list or file to not annotate
 349        ignored_clips : set, optional
 350            clip ids to ignore
 351        annotation_suffix : str | set, optional
 352            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
 353            (not passed if creating from key objects or if irrelevant for the dataset)
 354        annotation_path : str | set, optional
 355            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
 356            from key objects)
 357        behavior_file : str, optional
 358            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
 359        correction : dict, optional
 360            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
 361            can be used to correct for variations in naming or to merge several labels in one
 362        frame_limit : int, default 0
 363            the smallest possible length of a clip (shorter clips are discarded)
 364        filter_annotated : bool, default False
 365            if True, the samples that do not have any labels will be filtered
 366        filter_background : bool, default False
 367            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
 368        error_class : str, optional
 369            the name of the error class (the annotations that intersect with this label will be discarded)
 370        min_frames_action : int, default 0
 371            the minimum length of an action (shorter actions are not annotated)
 372        key_objects : tuple, optional
 373            the key objects to load the AnnotationStore from
 374        visibility_min_score : float, default 5
 375            the minimum visibility score for visibility filtering
 376        visibility_min_frac : float, default 0.7
 377            the minimum fraction of visible frames for visibility filtering
 378        mask : dict, optional
 379            a masked value dictionary (for active learning simulation experiments)
 380        use_hard_negatives : bool, default False
 381            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
 382        interactive : bool, default False
 383            if `True`, annotation is assigned to pairs of individuals
 384        """
 385
 386        super().__init__()
 387
 388        if ignored_clips is None:
 389            ignored_clips = []
 390        self.len_segment = int(len_segment)
 391        self.exclusive = exclusive
 392        if overlap < 1:
 393            overlap = overlap * len_segment
 394        self.overlap = int(overlap)
 395        self.video_order = video_order
 396        self.min_frames = min_frames
 397        self.max_frames = max_frames
 398        self.visibility = visibility
 399        self.vis_min_score = visibility_min_score
 400        self.vis_min_frac = visibility_min_frac
 401        self.mask = mask
 402        self.use_negatives = use_hard_negatives
 403        self.interactive = interactive
 404        self.ignored_clips = ignored_clips
 405        self.file_paths = self._get_file_paths(annotation_path)
 406        self.ignored_classes = ignored_classes
 407        self.update_behaviors = False
 408
 409        self.ram = True
 410        self.original_coordinates = []
 411        self.filtered = []
 412
 413        self.step = self.len_segment - self.overlap
 414
 415        self.ann_suffix = annotation_suffix
 416        self.annotation_folder = annotation_path
 417        self.filter_annotated = filter_annotated
 418        self.filter_background = filter_background
 419        self.frame_limit = frame_limit
 420        self.min_frames_action = min_frames_action
 421        self.error_class = error_class
 422
 423        if correction is None:
 424            correction = {}
 425        self.correction = correction
 426
 427        if self.max_frames is None:
 428            self.max_frames = defaultdict(lambda: {})
 429        if self.min_frames is None:
 430            self.min_frames = defaultdict(lambda: {})
 431
 432        lists = [self.annotation_folder, self.ann_suffix]
 433        for i in range(len(lists)):
 434            iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str))
 435            if lists[i] is not None:
 436                if not iterable:
 437                    lists[i] = [lists[i]]
 438                lists[i] = [x for x in lists[i]]
 439        self.annotation_folder, self.ann_suffix = lists
 440
 441        if ignored_classes is None:
 442            ignored_classes = []
 443        self.ignored_classes = ignored_classes
 444        self._set_behaviors(behaviors, ignored_classes, behavior_file)
 445
 446        if key_objects is None and self.video_order is not None:
 447            self.data = self._load_data()
 448        elif key_objects is not None:
 449            self.load_from_key_objects(key_objects)
 450        else:
 451            self.data = None
 452        self.labeled_indices, self.unlabeled_indices = self._compute_labeled()
 453
 454    def __getitem__(self, ind):
 455        if self.data is None:
 456            raise RuntimeError("The annotation store data has not been initialized!")
 457        return self.data[ind]
 458
 459    def __len__(self) -> int:
 460        if self.data is None:
 461            raise RuntimeError("The annotation store data has not been initialized!")
 462        return len(self.data)
 463
 464    def remove(self, indices: List) -> None:
 465        """
 466        Remove the samples corresponding to indices
 467
 468        Parameters
 469        ----------
 470        indices : list
 471            a list of integer indices to remove
 472        """
 473
 474        if len(indices) > 0:
 475            mask = np.ones(len(self.data))
 476            mask[indices] = 0
 477            mask = mask.astype(bool)
 478            self.data = self.data[mask]
 479            self.original_coordinates = self.original_coordinates[mask]
 480
 481    def key_objects(self) -> Tuple:
 482        """
 483        Return a tuple of the key objects necessary to re-create the Store
 484
 485        Returns
 486        -------
 487        key_objects : tuple
 488            a tuple of key objects
 489        """
 490
 491        return (
 492            self.original_coordinates,
 493            self.data,
 494            self.behaviors,
 495            self.exclusive,
 496            self.len_segment,
 497            self.step,
 498            self.overlap,
 499        )
 500
 501    def load_from_key_objects(self, key_objects: Tuple) -> None:
 502        """
 503        Load the information from a tuple of key objects
 504
 505        Parameters
 506        ----------
 507        key_objects : tuple
 508            a tuple of key objects
 509        """
 510
 511        (
 512            self.original_coordinates,
 513            self.data,
 514            self.behaviors,
 515            self.exclusive,
 516            self.len_segment,
 517            self.step,
 518            self.overlap,
 519        ) = key_objects
 520        self.labeled_indices, self.unlabeled_indices = self._compute_labeled()
 521
 522    def to_ram(self) -> None:
 523        """
 524        Transfer the data samples to RAM if they were previously stored as file paths
 525        """
 526
 527        pass
 528
 529    def get_original_coordinates(self) -> np.ndarray:
 530        """
 531        Return the video_indices array
 532
 533        Returns
 534        -------
 535        original_coordinates : numpy.ndarray
 536            an array that contains the coordinates of the data samples in original input data
 537        """
 538        return self.original_coordinates
 539
 540    def create_subsample(self, indices: List, ssl_indices: List = None):
 541        """
 542        Create a new store that contains a subsample of the data
 543
 544        Parameters
 545        ----------
 546        indices : list
 547            the indices to be included in the subsample
 548        ssl_indices : list, optional
 549            the indices to be included in the subsample without the annotation data
 550        """
 551
 552        if ssl_indices is None:
 553            ssl_indices = []
 554        data = copy(self.data)
 555        data[ssl_indices, ...] = -100
 556        new = self.new()
 557        new.original_coordinates = self.original_coordinates[indices + ssl_indices]
 558        new.data = self.data[indices + ssl_indices]
 559        new.labeled_indices, new.unlabeled_indices = new._compute_labeled()
 560        new.behaviors = self.behaviors
 561        new.exclusive = self.exclusive
 562        new.len_segment = self.len_segment
 563        new.step = self.step
 564        new.overlap = self.overlap
 565        new.max_frames = self.max_frames
 566        new.min_frames = self.min_frames
 567        return new
 568
 569    def get_len(self, return_unlabeled: bool) -> int:
 570        """
 571        Get the length of the subsample of labeled/unlabeled data
 572
 573        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
 574        and if return_unlabeled is None the index is already correct
 575
 576        Parameters
 577        ----------
 578        return_unlabeled : bool
 579            the identifier for the subsample
 580
 581        Returns
 582        -------
 583        length : int
 584            the length of the subsample
 585        """
 586
 587        if self.data is None:
 588            raise RuntimeError("The annotation store data has not been initialized!")
 589        elif return_unlabeled is None:
 590            return len(self.data)
 591        elif return_unlabeled:
 592            return len(self.unlabeled_indices)
 593        else:
 594            return len(self.labeled_indices)
 595
 596    def get_indices(self, return_unlabeled: bool) -> List:
 597        """
 598        Get a list of indices of samples in the labeled/unlabeled subset
 599
 600        Parameters
 601        ----------
 602        return_unlabeled : bool
 603            the identifier for the subsample (`True` for unlabeled, `False` for labeled, `None` for the
 604            whole dataset)
 605
 606        Returns
 607        -------
 608        indices : list
 609            a list of indices that meet the criteria
 610        """
 611
 612        return list(range(len(self.data)))
 613
 614    def count_classes(
 615        self, perc: bool = False, zeros: bool = False, bouts: bool = False
 616    ) -> Dict:
 617        """
 618        Get a dictionary with class-wise frame counts
 619
 620        Parameters
 621        ----------
 622        perc : bool, default False
 623            if `True`, a fraction of the total frame count is returned
 624        zeros : bool, default False
 625            if `True` and annotation is not exclusive, zero counts are returned
 626        bouts : bool, default False
 627            if `True`, instead of frame counts segment counts are returned
 628
 629        Returns
 630        -------
 631        count_dictionary : dict
 632            a dictionary with class indices as keys and frame counts as values
 633        """
 634
 635        if bouts:
 636            if self.overlap != 0:
 637                data = {}
 638                for video, value in self.max_frames.items():
 639                    for clip, end in value.items():
 640                        length = end - self._get_min_frame(video, clip)
 641                        if self.exclusive:
 642                            data[f"{video}---{clip}"] = -100 * torch.ones(length)
 643                        else:
 644                            data[f"{video}---{clip}"] = -100 * torch.ones(
 645                                (len(self.behaviors_dict()), length)
 646                            )
 647                for x, coords in zip(self.data, self.original_coordinates):
 648                    split = coords[0].split("---")
 649                    l = self._get_max_frame(split[0], split[1]) - self._get_min_frame(
 650                        split[0], split[1]
 651                    )
 652                    i = coords[1]
 653                    start = int(i) * self.step
 654                    end = min(start + self.len_segment, l)
 655                    data[coords[0]][..., start:end] = x[..., : end - start]
 656                values = []
 657                for key, value in data.items():
 658                    values.append(value)
 659                    values.append(-100 * torch.ones((*value.shape[:-1], 1)))
 660                data = torch.cat(values, -1).T
 661            else:
 662                data = copy(self.data)
 663                if self.exclusive:
 664                    data = data.flatten()
 665                else:
 666                    data = data.transpose(1, 2).reshape(-1, len(self.behaviors))
 667            count_dictionary = {}
 668            for c in self.behaviors_dict():
 669                if self.exclusive:
 670                    arr = data == c
 671                else:
 672                    if zeros:
 673                        arr = data[:, c] == 0
 674                    else:
 675                        arr = data[:, c] == 1
 676                output, indices = torch.unique_consecutive(arr, return_inverse=True)
 677                true_indices = torch.where(output)[0]
 678                count_dictionary[c] = len(true_indices)
 679        else:
 680            ind = 1
 681            if zeros:
 682                ind = 0
 683            if self.exclusive:
 684                count_dictionary = dict(Counter(self.data.flatten().cpu().numpy()))
 685            else:
 686                d = {}
 687                for i in range(self.data.shape[1]):
 688                    cnt = Counter(self.data[:, i, :].flatten().cpu().numpy())
 689                    d[i] = cnt[ind]
 690                count_dictionary = d
 691            if perc:
 692                total = sum([v for k, v in count_dictionary.items()])
 693                count_dictionary = {k: v / total for k, v in count_dictionary.items()}
 694        for i in self.behaviors_dict():
 695            if i not in count_dictionary:
 696                count_dictionary[i] = 0
 697        return count_dictionary
 698
 699    def behaviors_dict(self) -> Dict:
 700        """
 701        Get a dictionary of class names
 702
 703        Returns
 704        -------
 705        behavior_dictionary: dict
 706            a dictionary with class indices as keys and class names as values
 707        """
 708
 709        # if self.behaviors is None
 710        if self.exclusive and "other" not in self.behaviors:
 711            d = {i + 1: b for i, b in enumerate(self.behaviors)}
 712            d[0] = "other"
 713        else:
 714            d = {i: b for i, b in enumerate(self.behaviors)}
 715        return d
 716
 717    def annotation_class(self) -> str:
 718        """
 719        Get the type of annotation ('exclusive_classification', 'nonexclusive_classification')
 720
 721        Returns
 722        -------
 723        annotation_class : str
 724            the type of annotation
 725        """
 726
 727        if self.exclusive:
 728            return "exclusive_classification"
 729        else:
 730            return "nonexclusive_classification"
 731
 732    def size(self) -> int:
 733        """
 734        Get the total number of frames in the data
 735
 736        Returns
 737        -------
 738        size : int
 739            the total number of frames
 740        """
 741
 742        return self.data.shape[0] * self.data.shape[-1]
 743
 744    def filtered_indices(self) -> List:
 745        """
 746        Return the indices of the samples that should be removed
 747
 748        Choosing the indices can be based on any kind of filering defined in the __init__ function by the data
 749        parameters
 750
 751        Returns
 752        -------
 753        indices_to_remove : list
 754            a list of integer indices that should be removed
 755        """
 756
 757        return self.filtered
 758
 759    def set_pseudo_labels(self, labels: torch.Tensor) -> None:
 760        """
 761        Set pseudo labels to the unlabeled data
 762
 763        Parameters
 764        ----------
 765        labels : torch.Tensor
 766            a tensor of pseudo-labels for the unlabeled data
 767        """
 768
 769        self.data[self.unlabeled_indices] = labels
 770
 771    @classmethod
 772    def get_file_ids(
 773        cls,
 774        annotation_path: Union[str, Set],
 775        annotation_suffix: Union[str, Set],
 776        *args,
 777        **kwargs,
 778    ) -> List:
 779        """
 780        Process data parameters and return a list of ids  of the videos that should
 781        be processed by the __init__ function
 782
 783        Parameters
 784        ----------
 785        annotation_path : str | set
 786            the path or the set of paths to the folder where the annotation files are stored
 787        annotation_suffix : str | set, optional
 788            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
 789
 790        Returns
 791        -------
 792        video_ids : list
 793            a list of video file ids
 794        """
 795
 796        lists = [annotation_path, annotation_suffix]
 797        for i in range(len(lists)):
 798            iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str))
 799            if lists[i] is not None:
 800                if not iterable:
 801                    lists[i] = [lists[i]]
 802                lists[i] = [x for x in lists[i]]
 803        annotation_path, annotation_suffix = lists
 804        files = []
 805        for folder in annotation_path:
 806            files += [
 807                strip_suffix(os.path.basename(file), annotation_suffix)
 808                for file in os.listdir(folder)
 809                if file.endswith(tuple([x for x in annotation_suffix]))
 810            ]
 811        files = sorted(files, key=lambda x: os.path.basename(x))
 812        return files
 813
 814    def _set_behaviors(
 815        self, behaviors: List, ignored_classes: List, behavior_file: str
 816    ):
 817        """
 818        Get a list of behaviors that should be annotated from behavior parameters
 819        """
 820
 821        if behaviors is not None:
 822            for b in ignored_classes:
 823                if b in behaviors:
 824                    behaviors.remove(b)
 825        self.behaviors = behaviors
 826
 827    def _compute_labeled(self) -> Tuple[torch.Tensor, torch.Tensor]:
 828        """
 829        Get the indices of labeled (annotated) and unlabeled samples
 830        """
 831
 832        if self.data is not None and len(self.data) > 0:
 833            unlabeled = torch.sum(self.data != -100, dim=1) == 0
 834            labeled_indices = torch.where(~unlabeled)[0]
 835            unlabeled_indices = torch.where(unlabeled)[0]
 836        else:
 837            labeled_indices, unlabeled_indices = torch.tensor([]), torch.tensor([])
 838        return labeled_indices, unlabeled_indices
 839
 840    def _generate_annotation(self, times: Dict, name: str) -> Dict:
 841        """
 842        Process a loaded annotation file to generate a training labels dictionary
 843        """
 844
 845        annotation = {}
 846        if self.behaviors is None and times is not None:
 847            behaviors = set()
 848            for d in times.values():
 849                behaviors.update([k for k, v in d.items()])
 850            self.behaviors = [
 851                x
 852                for x in sorted(behaviors)
 853                if x not in self.ignored_classes
 854                and not x.startswith("negative")
 855                and not x.startswith("unknown")
 856            ]
 857        elif self.behaviors is None and times is None:
 858            raise ValueError("Cannot generate annotqtion without behavior information!")
 859        beh_inv = {v: k for k, v in self.behaviors_dict().items()}
 860        # if there is no annotation file, generate empty annotation
 861        if self.interactive:
 862            clips = [
 863                "+".join(sorted(x))
 864                for x in combinations(self.max_frames[name].keys(), 2)
 865            ]
 866        else:
 867            clips = list(self.max_frames[name].keys())
 868        if times is None:
 869            clips = [x for x in clips if x not in self.ignored_clips]
 870        # otherwise, apply filters and generate label arrays
 871        else:
 872            clips = [
 873                x
 874                for x in clips
 875                if x not in self.ignored_clips and x not in times.keys()
 876            ]
 877            for ind in times.keys():
 878                try:
 879                    min_frame = self._get_min_frame(name, ind)
 880                    max_frame = self._get_max_frame(name, ind)
 881                except KeyError:
 882                    continue
 883                go_on = max_frame - min_frame + 1 >= self.frame_limit
 884                if go_on:
 885                    v_len = max_frame - min_frame + 1
 886                    if self.exclusive:
 887                        if not self.filter_background:
 888                            value = beh_inv.get("other", 0)
 889                            labels = np.ones(v_len, dtype=np.compat.long) * value
 890                        else:
 891                            labels = -100 * np.ones(v_len, dtype=np.compat.long)
 892                    else:
 893                        labels = np.zeros((len(self.behaviors), v_len), dtype=np.float)
 894                    cat_new = []
 895                    for cat in times[ind].keys():
 896                        if cat.startswith("unknown"):
 897                            cat_new.append(cat)
 898                    for cat in times[ind].keys():
 899                        if cat.startswith("negative"):
 900                            cat_new.append(cat)
 901                    for cat in times[ind].keys():
 902                        if not cat.startswith("negative") and not cat.startswith(
 903                            "unknown"
 904                        ):
 905                            cat_new.append(cat)
 906                    for cat in cat_new:
 907                        neg = False
 908                        unknown = False
 909                        cat_times = times[ind][cat]
 910                        if self.use_negatives and cat.startswith("negative"):
 911                            cat = " ".join(cat.split()[1:])
 912                            neg = True
 913                        elif cat.startswith("unknown"):
 914                            cat = " ".join(cat.split()[1:])
 915                            unknown = True
 916                        if cat in self.correction:
 917                            cat = self.correction[cat]
 918                        for start, end, amb in cat_times:
 919                            if end > self._get_max_frame(name, ind) + 1:
 920                                end = self._get_max_frame(name, ind) + 1
 921                            if amb != 0:
 922                                continue
 923                            start -= min_frame
 924                            end -= min_frame
 925                            if (
 926                                self.min_frames_action is not None
 927                                and end - start < self.min_frames_action
 928                            ):
 929                                continue
 930                            if (
 931                                self.vis_min_frac > 0
 932                                and self.vis_min_score > 0
 933                                and self.visibility is not None
 934                            ):
 935                                s = 0
 936                                for ind_k in ind.split("+"):
 937                                    s += np.sum(
 938                                        self.visibility[name][ind_k][start:end]
 939                                        > self.vis_min_score
 940                                    )
 941                                if s < self.vis_min_frac * (end - start) * len(
 942                                    ind.split("+")
 943                                ):
 944                                    continue
 945                            if cat in beh_inv:
 946                                cat_i_global = beh_inv[cat]
 947                                if self.exclusive:
 948                                    labels[start:end] = cat_i_global
 949                                else:
 950                                    if unknown:
 951                                        labels[cat_i_global, start:end] = -100
 952                                    elif neg:
 953                                        labels[cat_i_global, start:end] = 2
 954                                    else:
 955                                        labels[cat_i_global, start:end] = 1
 956                            else:
 957                                self.not_found.add(cat)
 958                                if self.filter_background:
 959                                    if not self.exclusive:
 960                                        labels[:, start:end][
 961                                            labels[:, start:end] == 0
 962                                        ] = 3
 963                                    else:
 964                                        labels[start:end][labels[start:end] == -100] = 0
 965
 966                    if self.error_class is not None and self.error_class in times[ind]:
 967                        for start, end, amb in times[ind][self.error_class]:
 968                            if self.exclusive:
 969                                labels[start:end] = -100
 970                            else:
 971                                labels[:, start:end] = -100
 972                    annotation[os.path.basename(name) + "---" + str(ind)] = labels
 973        for ind in clips:
 974            try:
 975                min_frame = self._get_min_frame(name, ind)
 976                max_frame = self._get_max_frame(name, ind)
 977            except KeyError:
 978                continue
 979            go_on = max_frame - min_frame + 1 >= self.frame_limit
 980            if go_on:
 981                v_len = max_frame - min_frame + 1
 982                if self.exclusive:
 983                    annotation[
 984                        os.path.basename(name) + "---" + str(ind)
 985                    ] = -100 * np.ones(v_len, dtype=np.compat.long)
 986                else:
 987                    annotation[
 988                        os.path.basename(name) + "---" + str(ind)
 989                    ] = -100 * np.ones((len(self.behaviors), v_len), dtype=np.float)
 990        return annotation
 991
 992    def _make_trimmed_annotations(self, annotations_dict: Dict) -> torch.Tensor:
 993        """
 994        Cut a label dictionary into overlapping pieces of equal length
 995        """
 996
 997        labels = []
 998        self.original_coordinates = []
 999        masked_all = []
1000        for v_id in sorted(annotations_dict.keys()):
1001            if v_id in annotations_dict:
1002                annotations = annotations_dict[v_id]
1003            else:
1004                raise ValueError(
1005                    f'The id list in {v_id.split("---")[0]} is not consistent across files'
1006                )
1007            split = v_id.split("---")
1008            if len(split) > 1:
1009                video_id, ind = split
1010            else:
1011                video_id = split[0]
1012                ind = ""
1013            min_frame = self._get_min_frame(video_id, ind)
1014            max_frame = self._get_max_frame(video_id, ind)
1015            v_len = max_frame - min_frame + 1
1016            sp = np.arange(0, v_len, self.step)
1017            pad = sp[-1] + self.len_segment - v_len
1018            if self.exclusive:
1019                annotations = np.pad(annotations, ((0, pad)), constant_values=-100)
1020            else:
1021                annotations = np.pad(
1022                    annotations, ((0, 0), (0, pad)), constant_values=-100
1023                )
1024            masked = np.zeros(annotations.shape)
1025            if (
1026                self.mask is not None
1027                and video_id in self.mask["masked"]
1028                and ind in self.mask["masked"][video_id]
1029            ):
1030                for start, end in self.mask["masked"][video_id][ind]:
1031                    masked[..., int(start) : int(end)] = 1
1032            for i, start in enumerate(sp):
1033                self.original_coordinates.append((v_id, i))
1034                if self.exclusive:
1035                    ann = annotations[start : start + self.len_segment]
1036                    m = masked[start : start + self.len_segment]
1037                else:
1038                    ann = annotations[:, start : start + self.len_segment]
1039                    m = masked[:, start : start + self.len_segment]
1040                labels.append(ann)
1041                masked_all.append(m)
1042        self.original_coordinates = np.array(self.original_coordinates)
1043        labels = torch.tensor(np.array(labels))
1044        masked_all = torch.tensor(np.array(masked_all)).int().bool()
1045        if self.filter_background and not self.exclusive:
1046            for i, label in enumerate(labels):
1047                label[:, torch.sum((label == 1) | (label == 3), 0) == 0] = -100
1048                label[label == 3] = 0
1049        labels[(labels != -100) & masked_all] = -200
1050        return labels
1051
1052    @classmethod
1053    def _get_file_paths(cls, annotation_path: Union[str, Set]) -> List:
1054        """
1055        Get a list of relevant files
1056        """
1057
1058        file_paths = []
1059        if annotation_path is not None:
1060            if isinstance(annotation_path, str):
1061                annotation_path = [annotation_path]
1062            for folder in annotation_path:
1063                file_paths += [os.path.join(folder, x) for x in os.listdir(folder)]
1064        return file_paths
1065
1066    def _get_max_frame(self, video_id: str, clip_id: str):
1067        """
1068        Get the end frame of a clip in a video
1069        """
1070
1071        if clip_id in self.max_frames[video_id]:
1072            return self.max_frames[video_id][clip_id]
1073        else:
1074            return min(
1075                [self.max_frames[video_id][ind_k] for ind_k in clip_id.split("+")]
1076            )
1077
1078    def _get_min_frame(self, video_id, clip_id):
1079        """
1080        Get the start frame of a clip in a video
1081        """
1082
1083        if clip_id in self.min_frames[video_id]:
1084            return self.min_frames[video_id][clip_id]
1085        else:
1086            return max(
1087                [self.min_frames[video_id][ind_k] for ind_k in clip_id.split("+")]
1088            )
1089
1090    @abstractmethod
1091    def _load_data(self) -> torch.Tensor:
1092        """
1093        Load behavior annotation and generate annotation prompts
1094        """

A general realization of an annotation store for action segmentation tasks

Assumes the following file structure:

annotation_path
├── video1_annotation.pickle
└── video2_labels.pickle

Here annotation_suffix is {'_annotation.pickle', '_labels.pickle'}.

ActionSegmentationStore( video_order: List = None, min_frames: Dict = None, max_frames: Dict = None, visibility: Dict = None, exclusive: bool = True, len_segment: int = 128, overlap: int = 0, behaviors: Set = None, ignored_classes: Set = None, ignored_clips: Set = None, annotation_suffix: Union[Set, str] = None, annotation_path: Union[Set, str] = None, behavior_file: str = None, correction: Dict = None, frame_limit: int = 0, filter_annotated: bool = False, filter_background: bool = False, error_class: str = None, min_frames_action: int = None, key_objects: Tuple = None, visibility_min_score: float = 0.2, visibility_min_frac: float = 0.7, mask: Dict = None, use_hard_negatives: bool = False, interactive: bool = False, *args, **kwargs)
294    def __init__(
295        self,
296        video_order: List = None,
297        min_frames: Dict = None,
298        max_frames: Dict = None,
299        visibility: Dict = None,
300        exclusive: bool = True,
301        len_segment: int = 128,
302        overlap: int = 0,
303        behaviors: Set = None,
304        ignored_classes: Set = None,
305        ignored_clips: Set = None,
306        annotation_suffix: Union[Set, str] = None,
307        annotation_path: Union[Set, str] = None,
308        behavior_file: str = None,
309        correction: Dict = None,
310        frame_limit: int = 0,
311        filter_annotated: bool = False,
312        filter_background: bool = False,
313        error_class: str = None,
314        min_frames_action: int = None,
315        key_objects: Tuple = None,
316        visibility_min_score: float = 0.2,
317        visibility_min_frac: float = 0.7,
318        mask: Dict = None,
319        use_hard_negatives: bool = False,
320        interactive: bool = False,
321        *args,
322        **kwargs,
323    ) -> None:
324        """
325        Parameters
326        ----------
327        video_order : list, optional
328            a list of video ids that should be processed in the same order (not passed if creating from key objects)
329        min_frames : dict, optional
330            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
331            clip start frames (not passed if creating from key objects)
332        max_frames : dict, optional
333            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
334            clip end frames (not passed if creating from key objects)
335        visibility : dict, optional
336            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
337            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
338        exclusive : bool, default True
339            if True, the annotation is single-label; if False, multi-label
340        len_segment : int, default 128
341            the length of the segments in which the data should be cut (in frames)
342        overlap : int, default 0
343            the length of the overlap between neighboring segments (in frames)
344        behaviors : set, optional
345            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
346            loaded from a file)
347        ignored_classes : set, optional
348            the list of behaviors from the behaviors list or file to not annotate
349        ignored_clips : set, optional
350            clip ids to ignore
351        annotation_suffix : str | set, optional
352            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
353            (not passed if creating from key objects or if irrelevant for the dataset)
354        annotation_path : str | set, optional
355            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
356            from key objects)
357        behavior_file : str, optional
358            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
359        correction : dict, optional
360            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
361            can be used to correct for variations in naming or to merge several labels in one
362        frame_limit : int, default 0
363            the smallest possible length of a clip (shorter clips are discarded)
364        filter_annotated : bool, default False
365            if True, the samples that do not have any labels will be filtered
366        filter_background : bool, default False
367            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
368        error_class : str, optional
369            the name of the error class (the annotations that intersect with this label will be discarded)
370        min_frames_action : int, default 0
371            the minimum length of an action (shorter actions are not annotated)
372        key_objects : tuple, optional
373            the key objects to load the AnnotationStore from
374        visibility_min_score : float, default 5
375            the minimum visibility score for visibility filtering
376        visibility_min_frac : float, default 0.7
377            the minimum fraction of visible frames for visibility filtering
378        mask : dict, optional
379            a masked value dictionary (for active learning simulation experiments)
380        use_hard_negatives : bool, default False
381            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
382        interactive : bool, default False
383            if `True`, annotation is assigned to pairs of individuals
384        """
385
386        super().__init__()
387
388        if ignored_clips is None:
389            ignored_clips = []
390        self.len_segment = int(len_segment)
391        self.exclusive = exclusive
392        if overlap < 1:
393            overlap = overlap * len_segment
394        self.overlap = int(overlap)
395        self.video_order = video_order
396        self.min_frames = min_frames
397        self.max_frames = max_frames
398        self.visibility = visibility
399        self.vis_min_score = visibility_min_score
400        self.vis_min_frac = visibility_min_frac
401        self.mask = mask
402        self.use_negatives = use_hard_negatives
403        self.interactive = interactive
404        self.ignored_clips = ignored_clips
405        self.file_paths = self._get_file_paths(annotation_path)
406        self.ignored_classes = ignored_classes
407        self.update_behaviors = False
408
409        self.ram = True
410        self.original_coordinates = []
411        self.filtered = []
412
413        self.step = self.len_segment - self.overlap
414
415        self.ann_suffix = annotation_suffix
416        self.annotation_folder = annotation_path
417        self.filter_annotated = filter_annotated
418        self.filter_background = filter_background
419        self.frame_limit = frame_limit
420        self.min_frames_action = min_frames_action
421        self.error_class = error_class
422
423        if correction is None:
424            correction = {}
425        self.correction = correction
426
427        if self.max_frames is None:
428            self.max_frames = defaultdict(lambda: {})
429        if self.min_frames is None:
430            self.min_frames = defaultdict(lambda: {})
431
432        lists = [self.annotation_folder, self.ann_suffix]
433        for i in range(len(lists)):
434            iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str))
435            if lists[i] is not None:
436                if not iterable:
437                    lists[i] = [lists[i]]
438                lists[i] = [x for x in lists[i]]
439        self.annotation_folder, self.ann_suffix = lists
440
441        if ignored_classes is None:
442            ignored_classes = []
443        self.ignored_classes = ignored_classes
444        self._set_behaviors(behaviors, ignored_classes, behavior_file)
445
446        if key_objects is None and self.video_order is not None:
447            self.data = self._load_data()
448        elif key_objects is not None:
449            self.load_from_key_objects(key_objects)
450        else:
451            self.data = None
452        self.labeled_indices, self.unlabeled_indices = self._compute_labeled()

Parameters

video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) behaviors : set, optional the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are loaded from a file) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate ignored_clips : set, optional clip ids to ignore annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} (not passed if creating from key objects or if irrelevant for the dataset) annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one frame_limit : int, default 0 the smallest possible length of a clip (shorter clips are discarded) filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) use_hard_negatives : bool, default False mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing interactive : bool, default False if True, annotation is assigned to pairs of individuals

def remove(self, indices: List) -> None:
464    def remove(self, indices: List) -> None:
465        """
466        Remove the samples corresponding to indices
467
468        Parameters
469        ----------
470        indices : list
471            a list of integer indices to remove
472        """
473
474        if len(indices) > 0:
475            mask = np.ones(len(self.data))
476            mask[indices] = 0
477            mask = mask.astype(bool)
478            self.data = self.data[mask]
479            self.original_coordinates = self.original_coordinates[mask]

Remove the samples corresponding to indices

Parameters

indices : list a list of integer indices to remove

def key_objects(self) -> Tuple:
481    def key_objects(self) -> Tuple:
482        """
483        Return a tuple of the key objects necessary to re-create the Store
484
485        Returns
486        -------
487        key_objects : tuple
488            a tuple of key objects
489        """
490
491        return (
492            self.original_coordinates,
493            self.data,
494            self.behaviors,
495            self.exclusive,
496            self.len_segment,
497            self.step,
498            self.overlap,
499        )

Return a tuple of the key objects necessary to re-create the Store

Returns

key_objects : tuple a tuple of key objects

def load_from_key_objects(self, key_objects: Tuple) -> None:
501    def load_from_key_objects(self, key_objects: Tuple) -> None:
502        """
503        Load the information from a tuple of key objects
504
505        Parameters
506        ----------
507        key_objects : tuple
508            a tuple of key objects
509        """
510
511        (
512            self.original_coordinates,
513            self.data,
514            self.behaviors,
515            self.exclusive,
516            self.len_segment,
517            self.step,
518            self.overlap,
519        ) = key_objects
520        self.labeled_indices, self.unlabeled_indices = self._compute_labeled()

Load the information from a tuple of key objects

Parameters

key_objects : tuple a tuple of key objects

def to_ram(self) -> None:
522    def to_ram(self) -> None:
523        """
524        Transfer the data samples to RAM if they were previously stored as file paths
525        """
526
527        pass

Transfer the data samples to RAM if they were previously stored as file paths

def get_original_coordinates(self) -> numpy.ndarray:
529    def get_original_coordinates(self) -> np.ndarray:
530        """
531        Return the video_indices array
532
533        Returns
534        -------
535        original_coordinates : numpy.ndarray
536            an array that contains the coordinates of the data samples in original input data
537        """
538        return self.original_coordinates

Return the video_indices array

Returns

original_coordinates : numpy.ndarray an array that contains the coordinates of the data samples in original input data

def create_subsample(self, indices: List, ssl_indices: List = None)
540    def create_subsample(self, indices: List, ssl_indices: List = None):
541        """
542        Create a new store that contains a subsample of the data
543
544        Parameters
545        ----------
546        indices : list
547            the indices to be included in the subsample
548        ssl_indices : list, optional
549            the indices to be included in the subsample without the annotation data
550        """
551
552        if ssl_indices is None:
553            ssl_indices = []
554        data = copy(self.data)
555        data[ssl_indices, ...] = -100
556        new = self.new()
557        new.original_coordinates = self.original_coordinates[indices + ssl_indices]
558        new.data = self.data[indices + ssl_indices]
559        new.labeled_indices, new.unlabeled_indices = new._compute_labeled()
560        new.behaviors = self.behaviors
561        new.exclusive = self.exclusive
562        new.len_segment = self.len_segment
563        new.step = self.step
564        new.overlap = self.overlap
565        new.max_frames = self.max_frames
566        new.min_frames = self.min_frames
567        return new

Create a new store that contains a subsample of the data

Parameters

indices : list the indices to be included in the subsample ssl_indices : list, optional the indices to be included in the subsample without the annotation data

def get_len(self, return_unlabeled: bool) -> int:
569    def get_len(self, return_unlabeled: bool) -> int:
570        """
571        Get the length of the subsample of labeled/unlabeled data
572
573        If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled
574        and if return_unlabeled is None the index is already correct
575
576        Parameters
577        ----------
578        return_unlabeled : bool
579            the identifier for the subsample
580
581        Returns
582        -------
583        length : int
584            the length of the subsample
585        """
586
587        if self.data is None:
588            raise RuntimeError("The annotation store data has not been initialized!")
589        elif return_unlabeled is None:
590            return len(self.data)
591        elif return_unlabeled:
592            return len(self.unlabeled_indices)
593        else:
594            return len(self.labeled_indices)

Get the length of the subsample of labeled/unlabeled data

If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct

Parameters

return_unlabeled : bool the identifier for the subsample

Returns

length : int the length of the subsample

def get_indices(self, return_unlabeled: bool) -> List:
596    def get_indices(self, return_unlabeled: bool) -> List:
597        """
598        Get a list of indices of samples in the labeled/unlabeled subset
599
600        Parameters
601        ----------
602        return_unlabeled : bool
603            the identifier for the subsample (`True` for unlabeled, `False` for labeled, `None` for the
604            whole dataset)
605
606        Returns
607        -------
608        indices : list
609            a list of indices that meet the criteria
610        """
611
612        return list(range(len(self.data)))

Get a list of indices of samples in the labeled/unlabeled subset

Parameters

return_unlabeled : bool the identifier for the subsample (True for unlabeled, False for labeled, None for the whole dataset)

Returns

indices : list a list of indices that meet the criteria

def count_classes( self, perc: bool = False, zeros: bool = False, bouts: bool = False) -> Dict:
614    def count_classes(
615        self, perc: bool = False, zeros: bool = False, bouts: bool = False
616    ) -> Dict:
617        """
618        Get a dictionary with class-wise frame counts
619
620        Parameters
621        ----------
622        perc : bool, default False
623            if `True`, a fraction of the total frame count is returned
624        zeros : bool, default False
625            if `True` and annotation is not exclusive, zero counts are returned
626        bouts : bool, default False
627            if `True`, instead of frame counts segment counts are returned
628
629        Returns
630        -------
631        count_dictionary : dict
632            a dictionary with class indices as keys and frame counts as values
633        """
634
635        if bouts:
636            if self.overlap != 0:
637                data = {}
638                for video, value in self.max_frames.items():
639                    for clip, end in value.items():
640                        length = end - self._get_min_frame(video, clip)
641                        if self.exclusive:
642                            data[f"{video}---{clip}"] = -100 * torch.ones(length)
643                        else:
644                            data[f"{video}---{clip}"] = -100 * torch.ones(
645                                (len(self.behaviors_dict()), length)
646                            )
647                for x, coords in zip(self.data, self.original_coordinates):
648                    split = coords[0].split("---")
649                    l = self._get_max_frame(split[0], split[1]) - self._get_min_frame(
650                        split[0], split[1]
651                    )
652                    i = coords[1]
653                    start = int(i) * self.step
654                    end = min(start + self.len_segment, l)
655                    data[coords[0]][..., start:end] = x[..., : end - start]
656                values = []
657                for key, value in data.items():
658                    values.append(value)
659                    values.append(-100 * torch.ones((*value.shape[:-1], 1)))
660                data = torch.cat(values, -1).T
661            else:
662                data = copy(self.data)
663                if self.exclusive:
664                    data = data.flatten()
665                else:
666                    data = data.transpose(1, 2).reshape(-1, len(self.behaviors))
667            count_dictionary = {}
668            for c in self.behaviors_dict():
669                if self.exclusive:
670                    arr = data == c
671                else:
672                    if zeros:
673                        arr = data[:, c] == 0
674                    else:
675                        arr = data[:, c] == 1
676                output, indices = torch.unique_consecutive(arr, return_inverse=True)
677                true_indices = torch.where(output)[0]
678                count_dictionary[c] = len(true_indices)
679        else:
680            ind = 1
681            if zeros:
682                ind = 0
683            if self.exclusive:
684                count_dictionary = dict(Counter(self.data.flatten().cpu().numpy()))
685            else:
686                d = {}
687                for i in range(self.data.shape[1]):
688                    cnt = Counter(self.data[:, i, :].flatten().cpu().numpy())
689                    d[i] = cnt[ind]
690                count_dictionary = d
691            if perc:
692                total = sum([v for k, v in count_dictionary.items()])
693                count_dictionary = {k: v / total for k, v in count_dictionary.items()}
694        for i in self.behaviors_dict():
695            if i not in count_dictionary:
696                count_dictionary[i] = 0
697        return count_dictionary

Get a dictionary with class-wise frame counts

Parameters

perc : bool, default False if True, a fraction of the total frame count is returned zeros : bool, default False if True and annotation is not exclusive, zero counts are returned bouts : bool, default False if True, instead of frame counts segment counts are returned

Returns

count_dictionary : dict a dictionary with class indices as keys and frame counts as values

def behaviors_dict(self) -> Dict:
699    def behaviors_dict(self) -> Dict:
700        """
701        Get a dictionary of class names
702
703        Returns
704        -------
705        behavior_dictionary: dict
706            a dictionary with class indices as keys and class names as values
707        """
708
709        # if self.behaviors is None
710        if self.exclusive and "other" not in self.behaviors:
711            d = {i + 1: b for i, b in enumerate(self.behaviors)}
712            d[0] = "other"
713        else:
714            d = {i: b for i, b in enumerate(self.behaviors)}
715        return d

Get a dictionary of class names

Returns

behavior_dictionary: dict a dictionary with class indices as keys and class names as values

def annotation_class(self) -> str:
717    def annotation_class(self) -> str:
718        """
719        Get the type of annotation ('exclusive_classification', 'nonexclusive_classification')
720
721        Returns
722        -------
723        annotation_class : str
724            the type of annotation
725        """
726
727        if self.exclusive:
728            return "exclusive_classification"
729        else:
730            return "nonexclusive_classification"

Get the type of annotation ('exclusive_classification', 'nonexclusive_classification')

Returns

annotation_class : str the type of annotation

def size(self) -> int:
732    def size(self) -> int:
733        """
734        Get the total number of frames in the data
735
736        Returns
737        -------
738        size : int
739            the total number of frames
740        """
741
742        return self.data.shape[0] * self.data.shape[-1]

Get the total number of frames in the data

Returns

size : int the total number of frames

def filtered_indices(self) -> List:
744    def filtered_indices(self) -> List:
745        """
746        Return the indices of the samples that should be removed
747
748        Choosing the indices can be based on any kind of filering defined in the __init__ function by the data
749        parameters
750
751        Returns
752        -------
753        indices_to_remove : list
754            a list of integer indices that should be removed
755        """
756
757        return self.filtered

Return the indices of the samples that should be removed

Choosing the indices can be based on any kind of filering defined in the __init__ function by the data parameters

Returns

indices_to_remove : list a list of integer indices that should be removed

def set_pseudo_labels(self, labels: torch.Tensor) -> None:
759    def set_pseudo_labels(self, labels: torch.Tensor) -> None:
760        """
761        Set pseudo labels to the unlabeled data
762
763        Parameters
764        ----------
765        labels : torch.Tensor
766            a tensor of pseudo-labels for the unlabeled data
767        """
768
769        self.data[self.unlabeled_indices] = labels

Set pseudo labels to the unlabeled data

Parameters

labels : torch.Tensor a tensor of pseudo-labels for the unlabeled data

@classmethod
def get_file_ids( cls, annotation_path: Union[str, Set], annotation_suffix: Union[str, Set], *args, **kwargs) -> List:
771    @classmethod
772    def get_file_ids(
773        cls,
774        annotation_path: Union[str, Set],
775        annotation_suffix: Union[str, Set],
776        *args,
777        **kwargs,
778    ) -> List:
779        """
780        Process data parameters and return a list of ids  of the videos that should
781        be processed by the __init__ function
782
783        Parameters
784        ----------
785        annotation_path : str | set
786            the path or the set of paths to the folder where the annotation files are stored
787        annotation_suffix : str | set, optional
788            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
789
790        Returns
791        -------
792        video_ids : list
793            a list of video file ids
794        """
795
796        lists = [annotation_path, annotation_suffix]
797        for i in range(len(lists)):
798            iterable = isinstance(lists[i], Iterable) * (not isinstance(lists[i], str))
799            if lists[i] is not None:
800                if not iterable:
801                    lists[i] = [lists[i]]
802                lists[i] = [x for x in lists[i]]
803        annotation_path, annotation_suffix = lists
804        files = []
805        for folder in annotation_path:
806            files += [
807                strip_suffix(os.path.basename(file), annotation_suffix)
808                for file in os.listdir(folder)
809                if file.endswith(tuple([x for x in annotation_suffix]))
810            ]
811        files = sorted(files, key=lambda x: os.path.basename(x))
812        return files

Process data parameters and return a list of ids of the videos that should be processed by the __init__ function

Parameters

annotation_path : str | set the path or the set of paths to the folder where the annotation files are stored annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}

Returns

video_ids : list a list of video file ids

class FileAnnotationStore(ActionSegmentationStore):
1097class FileAnnotationStore(ActionSegmentationStore):  # +
1098    """
1099    A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to one video
1100    """
1101
1102    def _generate_max_min_frames(self, times: Dict, video_id: str) -> None:
1103        """
1104        Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore`
1105        """
1106
1107        if video_id in self.max_frames:
1108            return
1109        for ind, cat_dict in times.items():
1110            maxes = []
1111            mins = []
1112            for cat, cat_list in cat_dict.items():
1113                if len(cat_list) > 0:
1114                    maxes.append(max([x[1] for x in cat_list]))
1115                    mins.append(min([x[0] for x in cat_list]))
1116            self.max_frames[video_id][ind] = max(maxes)
1117            self.min_frames[video_id][ind] = min(mins)
1118
1119    def _load_data(self) -> torch.Tensor:
1120        """
1121        Load behavior annotation and generate annotation prompts
1122        """
1123
1124        if self.video_order is None:
1125            return None
1126
1127        files = []
1128        for x in self.video_order:
1129            ok = False
1130            for folder in self.annotation_folder:
1131                for s in self.ann_suffix:
1132                    file = os.path.join(folder, x + s)
1133                    if os.path.exists(file):
1134                        files.append(file)
1135                        ok = True
1136                        break
1137            if not ok:
1138                files.append(None)
1139        self.not_found = set()
1140        annotations_dict = {}
1141        print("Computing annotation arrays...")
1142        for name, filename in tqdm(list(zip(self.video_order, files))):
1143            if filename is not None:
1144                times = self._open_annotations(filename)
1145            else:
1146                times = None
1147            if times is not None:
1148                self._generate_max_min_frames(times, name)
1149            annotations_dict.update(self._generate_annotation(times, name))
1150            del times
1151        annotation = self._make_trimmed_annotations(annotations_dict)
1152        del annotations_dict
1153        if self.filter_annotated:
1154            if self.exclusive:
1155                s = torch.sum((annotation != -100), dim=1)
1156            else:
1157                s = torch.sum(
1158                    torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1
1159                )
1160            self.filtered += torch.where(s == 0)[0].tolist()
1161        annotation[annotation == -200] = -100
1162        return annotation
1163
1164    @abstractmethod
1165    def _open_annotations(self, filename: str) -> Dict:
1166        """
1167        Load the annotation from filename
1168
1169        Parameters
1170        ----------
1171        filename : str
1172            path to an annotation file
1173
1174        Returns
1175        -------
1176        times : dict
1177            a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are
1178            lists of (start, end, ambiguity status) lists
1179        """

A generalized implementation of ActionSegmentationStore for datasets where one file corresponds to one video

class SequenceAnnotationStore(ActionSegmentationStore):
1182class SequenceAnnotationStore(ActionSegmentationStore):  # +
1183    """
1184    A generalized implementation of `ActionSegmentationStore` for datasets where one file corresponds to multiple videos
1185    """
1186
1187    def _generate_max_min_frames(self, times: Dict) -> None:
1188        """
1189        Generate `max_frames` and `min_frames` objects in case they were not passed from an `InputStore`
1190        """
1191
1192        for video_id in times:
1193            if video_id in self.max_frames:
1194                continue
1195            self.max_frames[video_id] = {}
1196            for ind, cat_dict in times[video_id].items():
1197                maxes = []
1198                mins = []
1199                for cat, cat_list in cat_dict.items():
1200                    maxes.append(max([x[1] for x in cat_list]))
1201                    mins.append(min([x[0] for x in cat_list]))
1202                self.max_frames[video_id][ind] = max(maxes)
1203                self.min_frames[video_id][ind] = min(mins)
1204
1205    @classmethod
1206    def get_file_ids(
1207        cls,
1208        filenames: List = None,
1209        annotation_path: str = None,
1210        *args,
1211        **kwargs,
1212    ) -> List:
1213        """
1214        Process data parameters and return a list of ids  of the videos that should
1215        be processed by the __init__ function
1216
1217        Parameters
1218        ----------
1219        filenames : list, optional
1220            a list of annotation file paths
1221        annotation_path : str, optional
1222            path to the annotation folder
1223
1224        Returns
1225        -------
1226        video_ids : list
1227            a list of video file ids
1228        """
1229
1230        file_paths = []
1231        if annotation_path is not None:
1232            if isinstance(annotation_path, str):
1233                annotation_path = [annotation_path]
1234            file_paths = []
1235            for folder in annotation_path:
1236                file_paths += [os.path.join(folder, x) for x in os.listdir(folder)]
1237        ids = set()
1238        for f in file_paths:
1239            if os.path.basename(f) in filenames:
1240                ids.add(os.path.basename(f))
1241        ids = sorted(ids)
1242        return ids
1243
1244    def _load_data(self) -> torch.Tensor:
1245        """
1246        Load behavior annotation and generate annotation prompts
1247        """
1248
1249        if self.video_order is None:
1250            return None
1251
1252        files = []
1253        for f in self.file_paths:
1254            if os.path.basename(f) in self.video_order:
1255                files.append(f)
1256        self.not_found = set()
1257        annotations_dict = {}
1258        for name, filename in tqdm(zip(self.video_order, files)):
1259            if filename is not None:
1260                times = self._open_sequences(filename)
1261            else:
1262                times = None
1263            if times is not None:
1264                self._generate_max_min_frames(times)
1265                none_ids = []
1266                for video_id, sequence_dict in times.items():
1267                    if sequence_dict is None:
1268                        none_ids.append(sequence_dict)
1269                        continue
1270                    annotations_dict.update(
1271                        self._generate_annotation(sequence_dict, video_id)
1272                    )
1273                for video_id in none_ids:
1274                    annotations_dict.update(self._generate_annotation(None, video_id))
1275                del times
1276        annotation = self._make_trimmed_annotations(annotations_dict)
1277        del annotations_dict
1278        if self.filter_annotated:
1279            if self.exclusive:
1280                s = torch.sum((annotation != -100), dim=1)
1281            else:
1282                s = torch.sum(
1283                    torch.sum((annotation != -100), dim=1) == annotation.shape[1], dim=1
1284                )
1285            self.filtered += torch.where(s == 0)[0].tolist()
1286        annotation[annotation == -200] = -100
1287        return annotation
1288
1289    @abstractmethod
1290    def _open_sequences(self, filename: str) -> Dict:
1291        """
1292        Load the annotation from filename
1293
1294        Parameters
1295        ----------
1296        filename : str
1297            path to an annotation file
1298
1299        Returns
1300        -------
1301        times : dict
1302            a nested dictionary where first-level keys are video ids, second-level keys are clip ids,
1303            third-level keys are categories and values are
1304            lists of (start, end, ambiguity status) lists
1305        """

A generalized implementation of ActionSegmentationStore for datasets where one file corresponds to multiple videos

@classmethod
def get_file_ids( cls, filenames: List = None, annotation_path: str = None, *args, **kwargs) -> List:
1205    @classmethod
1206    def get_file_ids(
1207        cls,
1208        filenames: List = None,
1209        annotation_path: str = None,
1210        *args,
1211        **kwargs,
1212    ) -> List:
1213        """
1214        Process data parameters and return a list of ids  of the videos that should
1215        be processed by the __init__ function
1216
1217        Parameters
1218        ----------
1219        filenames : list, optional
1220            a list of annotation file paths
1221        annotation_path : str, optional
1222            path to the annotation folder
1223
1224        Returns
1225        -------
1226        video_ids : list
1227            a list of video file ids
1228        """
1229
1230        file_paths = []
1231        if annotation_path is not None:
1232            if isinstance(annotation_path, str):
1233                annotation_path = [annotation_path]
1234            file_paths = []
1235            for folder in annotation_path:
1236                file_paths += [os.path.join(folder, x) for x in os.listdir(folder)]
1237        ids = set()
1238        for f in file_paths:
1239            if os.path.basename(f) in filenames:
1240                ids.add(os.path.basename(f))
1241        ids = sorted(ids)
1242        return ids

Process data parameters and return a list of ids of the videos that should be processed by the __init__ function

Parameters

filenames : list, optional a list of annotation file paths annotation_path : str, optional path to the annotation folder

Returns

video_ids : list a list of video file ids

class DLCAnnotationStore(FileAnnotationStore):
1308class DLCAnnotationStore(FileAnnotationStore):  # +
1309    """
1310    DLC type annotation data
1311
1312    The files are either the DLC2Action GUI output or a pickled dictionary of the following structure:
1313        - nested dictionary,
1314        - first-level keys are individual IDs,
1315        - second-level keys are labels,
1316        - values are lists of intervals,
1317        - the lists of intervals is formatted as `[start_frame, end_frame, ambiguity]`,
1318        - ambiguity is 1 if the action is ambiguous (!!at the moment DLC2Action will IGNORE those intervals!!) or 0 if it isn't.
1319
1320    A minimum working example of such a dictionary is:
1321    ```
1322    {
1323        "ind0": {},
1324        "ind1": {
1325            "running": [60, 70, 0]],
1326            "eating": []
1327        }
1328    }
1329    ```
1330
1331    Here there are two animals: `"ind0"` and `"ind1"`, and two actions: running and eating.
1332    The only annotated action is eating for `"ind1"` between frames 60 and 70.
1333
1334    If you generate those files manually, run this code for a sanity check:
1335    ```
1336    import pickle
1337
1338    with open("/path/to/annotation.pickle", "rb") as f:
1339    data = pickle.load(f)
1340
1341    for ind, ind_dict in data.items():
1342        print(f'individual {ind}:')
1343        for label, intervals in ind_dict.items():
1344            for start, end, ambiguity in intervals:
1345                if ambiguity == 0:
1346                    print(f'  from {start} to {end} frame: {label}')
1347    ```
1348
1349    Assumes the following file structure:
1350    ```
1351    annotation_path
1352    ├── video1_annotation.pickle
1353    └── video2_labels.pickle
1354    ```
1355    Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`.
1356    """
1357
1358    def _open_annotations(self, filename: str) -> Dict:
1359        """
1360        Load the annotation from `filename`
1361        """
1362
1363        try:
1364            with open(filename, "rb") as f:
1365                data = pickle.load(f)
1366            if isinstance(data, dict):
1367                annotation = data
1368                for ind in annotation:
1369                    for cat, cat_list in annotation[ind].items():
1370                        annotation[ind][cat] = [
1371                            [start, end, 0] for start, end in cat_list
1372                        ]
1373            else:
1374                _, loaded_labels, animals, loaded_times = data
1375                annotation = {}
1376                for ind, ind_list in zip(animals, loaded_times):
1377                    annotation[ind] = {}
1378                    for cat, cat_list in zip(loaded_labels, ind_list):
1379                        annotation[ind][cat] = cat_list
1380            return annotation
1381        except:
1382            print(f"{filename} is invalid or does not exist")
1383            return None

DLC type annotation data

The files are either the DLC2Action GUI output or a pickled dictionary of the following structure: - nested dictionary, - first-level keys are individual IDs, - second-level keys are labels, - values are lists of intervals, - the lists of intervals is formatted as [start_frame, end_frame, ambiguity], - ambiguity is 1 if the action is ambiguous (!!at the moment DLC2Action will IGNORE those intervals!!) or 0 if it isn't.

A minimum working example of such a dictionary is:

{
    "ind0": {},
    "ind1": {
        "running": [60, 70, 0]],
        "eating": []
    }
}

Here there are two animals: "ind0" and "ind1", and two actions: running and eating. The only annotated action is eating for "ind1" between frames 60 and 70.

If you generate those files manually, run this code for a sanity check:

import pickle

with open("/path/to/annotation.pickle", "rb") as f:
data = pickle.load(f)

for ind, ind_dict in data.items():
    print(f'individual {ind}:')
    for label, intervals in ind_dict.items():
        for start, end, ambiguity in intervals:
            if ambiguity == 0:
                print(f'  from {start} to {end} frame: {label}')

Assumes the following file structure:

annotation_path
├── video1_annotation.pickle
└── video2_labels.pickle

Here annotation_suffix is {'_annotation.pickle', '_labels.pickle'}.

class BorisAnnotationStore(FileAnnotationStore):
1386class BorisAnnotationStore(FileAnnotationStore):  # +
1387    """
1388    BORIS type annotation data
1389
1390    Assumes the following file structure:
1391    ```
1392    annotation_path
1393    ├── video1_annotation.pickle
1394    └── video2_labels.pickle
1395    ```
1396    Here `annotation_suffix` is `{'_annotation.pickle', '_labels.pickle'}`.
1397    """
1398
1399    def __init__(
1400        self,
1401        video_order: List = None,
1402        min_frames: Dict = None,
1403        max_frames: Dict = None,
1404        visibility: Dict = None,
1405        exclusive: bool = True,
1406        len_segment: int = 128,
1407        overlap: int = 0,
1408        behaviors: Set = None,
1409        ignored_classes: Set = None,
1410        annotation_suffix: Union[Set, str] = None,
1411        annotation_path: Union[Set, str] = None,
1412        behavior_file: str = None,
1413        correction: Dict = None,
1414        frame_limit: int = 0,
1415        filter_annotated: bool = False,
1416        filter_background: bool = False,
1417        error_class: str = None,
1418        min_frames_action: int = None,
1419        key_objects: Tuple = None,
1420        visibility_min_score: float = 0.2,
1421        visibility_min_frac: float = 0.7,
1422        mask: Dict = None,
1423        use_hard_negatives: bool = False,
1424        default_agent_name: str = "ind0",
1425        interactive: bool = False,
1426        ignored_clips: Set = None,
1427        *args,
1428        **kwargs,
1429    ) -> None:
1430        """
1431        Parameters
1432        ----------
1433        video_order : list, optional
1434            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1435        min_frames : dict, optional
1436            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1437            clip start frames (not passed if creating from key objects)
1438        max_frames : dict, optional
1439            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1440            clip end frames (not passed if creating from key objects)
1441        visibility : dict, optional
1442            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1443            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
1444        exclusive : bool, default True
1445            if True, the annotation is single-label; if False, multi-label
1446        len_segment : int, default 128
1447            the length of the segments in which the data should be cut (in frames)
1448        overlap : int, default 0
1449            the length of the overlap between neighboring segments (in frames)
1450        behaviors : set, optional
1451            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
1452            loaded from a file)
1453        ignored_classes : set, optional
1454            the list of behaviors from the behaviors list or file to not annotate
1455        annotation_suffix : str | set, optional
1456            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
1457            (not passed if creating from key objects or if irrelevant for the dataset)
1458        annotation_path : str | set, optional
1459            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1460            from key objects)
1461        behavior_file : str, optional
1462            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
1463        correction : dict, optional
1464            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
1465            can be used to correct for variations in naming or to merge several labels in one
1466        frame_limit : int, default 0
1467            the smallest possible length of a clip (shorter clips are discarded)
1468        filter_annotated : bool, default False
1469            if True, the samples that do not have any labels will be filtered
1470        filter_background : bool, default False
1471            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
1472        error_class : str, optional
1473            the name of the error class (the annotations that intersect with this label will be discarded)
1474        min_frames_action : int, default 0
1475            the minimum length of an action (shorter actions are not annotated)
1476        key_objects : tuple, optional
1477            the key objects to load the AnnotationStore from
1478        visibility_min_score : float, default 5
1479            the minimum visibility score for visibility filtering
1480        visibility_min_frac : float, default 0.7
1481            the minimum fraction of visible frames for visibility filtering
1482        mask : dict, optional
1483            a masked value dictionary (for active learning simulation experiments)
1484        use_hard_negatives : bool, default False
1485            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
1486        interactive : bool, default False
1487            if `True`, annotation is assigned to pairs of individuals
1488        ignored_clips : set, optional
1489            a set of clip ids to ignore
1490        """
1491
1492        self.default_agent_name = default_agent_name
1493        super().__init__(
1494            video_order=video_order,
1495            min_frames=min_frames,
1496            max_frames=max_frames,
1497            visibility=visibility,
1498            exclusive=exclusive,
1499            len_segment=len_segment,
1500            overlap=overlap,
1501            behaviors=behaviors,
1502            ignored_classes=ignored_classes,
1503            annotation_suffix=annotation_suffix,
1504            annotation_path=annotation_path,
1505            behavior_file=behavior_file,
1506            correction=correction,
1507            frame_limit=frame_limit,
1508            filter_annotated=filter_annotated,
1509            filter_background=filter_background,
1510            error_class=error_class,
1511            min_frames_action=min_frames_action,
1512            key_objects=key_objects,
1513            visibility_min_score=visibility_min_score,
1514            visibility_min_frac=visibility_min_frac,
1515            mask=mask,
1516            use_hard_negatives=use_hard_negatives,
1517            interactive=interactive,
1518            ignored_clips=ignored_clips,
1519        )
1520
1521    def _open_annotations(self, filename: str) -> Dict:
1522        """
1523        Load the annotation from filename
1524        """
1525
1526        try:
1527            df = pd.read_csv(filename, header=15)
1528            fps = df.iloc[0]["FPS"]
1529            df["Subject"] = df["Subject"].fillna(self.default_agent_name)
1530            loaded_labels = list(df["Behavior"].unique())
1531            animals = list(df["Subject"].unique())
1532            loaded_times = {}
1533            for ind in animals:
1534                loaded_times[ind] = {}
1535                agent_df = df[df["Subject"] == ind]
1536                for cat in loaded_labels:
1537                    filtered_df = agent_df[agent_df["Behavior"] == cat]
1538                    starts = (
1539                        filtered_df["Time"][filtered_df["Status"] == "START"] * fps
1540                    ).astype(int)
1541                    ends = (
1542                        filtered_df["Time"][filtered_df["Status"] == "STOP"] * fps
1543                    ).astype(int)
1544                    loaded_times[ind][cat] = [
1545                        [start, end, 0] for start, end in zip(starts, ends)
1546                    ]
1547            return loaded_times
1548        except:
1549            print(f"{filename} is invalid or does not exist")
1550            return None

BORIS type annotation data

Assumes the following file structure:

annotation_path
├── video1_annotation.pickle
└── video2_labels.pickle

Here annotation_suffix is {'_annotation.pickle', '_labels.pickle'}.

BorisAnnotationStore( video_order: List = None, min_frames: Dict = None, max_frames: Dict = None, visibility: Dict = None, exclusive: bool = True, len_segment: int = 128, overlap: int = 0, behaviors: Set = None, ignored_classes: Set = None, annotation_suffix: Union[Set, str] = None, annotation_path: Union[Set, str] = None, behavior_file: str = None, correction: Dict = None, frame_limit: int = 0, filter_annotated: bool = False, filter_background: bool = False, error_class: str = None, min_frames_action: int = None, key_objects: Tuple = None, visibility_min_score: float = 0.2, visibility_min_frac: float = 0.7, mask: Dict = None, use_hard_negatives: bool = False, default_agent_name: str = 'ind0', interactive: bool = False, ignored_clips: Set = None, *args, **kwargs)
1399    def __init__(
1400        self,
1401        video_order: List = None,
1402        min_frames: Dict = None,
1403        max_frames: Dict = None,
1404        visibility: Dict = None,
1405        exclusive: bool = True,
1406        len_segment: int = 128,
1407        overlap: int = 0,
1408        behaviors: Set = None,
1409        ignored_classes: Set = None,
1410        annotation_suffix: Union[Set, str] = None,
1411        annotation_path: Union[Set, str] = None,
1412        behavior_file: str = None,
1413        correction: Dict = None,
1414        frame_limit: int = 0,
1415        filter_annotated: bool = False,
1416        filter_background: bool = False,
1417        error_class: str = None,
1418        min_frames_action: int = None,
1419        key_objects: Tuple = None,
1420        visibility_min_score: float = 0.2,
1421        visibility_min_frac: float = 0.7,
1422        mask: Dict = None,
1423        use_hard_negatives: bool = False,
1424        default_agent_name: str = "ind0",
1425        interactive: bool = False,
1426        ignored_clips: Set = None,
1427        *args,
1428        **kwargs,
1429    ) -> None:
1430        """
1431        Parameters
1432        ----------
1433        video_order : list, optional
1434            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1435        min_frames : dict, optional
1436            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1437            clip start frames (not passed if creating from key objects)
1438        max_frames : dict, optional
1439            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1440            clip end frames (not passed if creating from key objects)
1441        visibility : dict, optional
1442            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1443            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
1444        exclusive : bool, default True
1445            if True, the annotation is single-label; if False, multi-label
1446        len_segment : int, default 128
1447            the length of the segments in which the data should be cut (in frames)
1448        overlap : int, default 0
1449            the length of the overlap between neighboring segments (in frames)
1450        behaviors : set, optional
1451            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
1452            loaded from a file)
1453        ignored_classes : set, optional
1454            the list of behaviors from the behaviors list or file to not annotate
1455        annotation_suffix : str | set, optional
1456            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
1457            (not passed if creating from key objects or if irrelevant for the dataset)
1458        annotation_path : str | set, optional
1459            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1460            from key objects)
1461        behavior_file : str, optional
1462            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
1463        correction : dict, optional
1464            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
1465            can be used to correct for variations in naming or to merge several labels in one
1466        frame_limit : int, default 0
1467            the smallest possible length of a clip (shorter clips are discarded)
1468        filter_annotated : bool, default False
1469            if True, the samples that do not have any labels will be filtered
1470        filter_background : bool, default False
1471            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
1472        error_class : str, optional
1473            the name of the error class (the annotations that intersect with this label will be discarded)
1474        min_frames_action : int, default 0
1475            the minimum length of an action (shorter actions are not annotated)
1476        key_objects : tuple, optional
1477            the key objects to load the AnnotationStore from
1478        visibility_min_score : float, default 5
1479            the minimum visibility score for visibility filtering
1480        visibility_min_frac : float, default 0.7
1481            the minimum fraction of visible frames for visibility filtering
1482        mask : dict, optional
1483            a masked value dictionary (for active learning simulation experiments)
1484        use_hard_negatives : bool, default False
1485            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
1486        interactive : bool, default False
1487            if `True`, annotation is assigned to pairs of individuals
1488        ignored_clips : set, optional
1489            a set of clip ids to ignore
1490        """
1491
1492        self.default_agent_name = default_agent_name
1493        super().__init__(
1494            video_order=video_order,
1495            min_frames=min_frames,
1496            max_frames=max_frames,
1497            visibility=visibility,
1498            exclusive=exclusive,
1499            len_segment=len_segment,
1500            overlap=overlap,
1501            behaviors=behaviors,
1502            ignored_classes=ignored_classes,
1503            annotation_suffix=annotation_suffix,
1504            annotation_path=annotation_path,
1505            behavior_file=behavior_file,
1506            correction=correction,
1507            frame_limit=frame_limit,
1508            filter_annotated=filter_annotated,
1509            filter_background=filter_background,
1510            error_class=error_class,
1511            min_frames_action=min_frames_action,
1512            key_objects=key_objects,
1513            visibility_min_score=visibility_min_score,
1514            visibility_min_frac=visibility_min_frac,
1515            mask=mask,
1516            use_hard_negatives=use_hard_negatives,
1517            interactive=interactive,
1518            ignored_clips=ignored_clips,
1519        )

Parameters

video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) behaviors : set, optional the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are loaded from a file) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} (not passed if creating from key objects or if irrelevant for the dataset) annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one frame_limit : int, default 0 the smallest possible length of a clip (shorter clips are discarded) filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) use_hard_negatives : bool, default False mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing interactive : bool, default False if True, annotation is assigned to pairs of individuals ignored_clips : set, optional a set of clip ids to ignore

class PKUMMDAnnotationStore(FileAnnotationStore):
1553class PKUMMDAnnotationStore(FileAnnotationStore):  # +
1554    """
1555    PKU-MMD annotation data
1556
1557    Assumes the following file structure:
1558    ```
1559    annotation_path
1560    ├── 0364-L.txt
1561    ...
1562    └── 0144-M.txt
1563    ```
1564    """
1565
1566    def __init__(
1567        self,
1568        video_order: List = None,
1569        min_frames: Dict = None,
1570        max_frames: Dict = None,
1571        visibility: Dict = None,
1572        exclusive: bool = True,
1573        len_segment: int = 128,
1574        overlap: int = 0,
1575        ignored_classes: Set = None,
1576        annotation_path: Union[Set, str] = None,
1577        behavior_file: str = None,
1578        correction: Dict = None,
1579        frame_limit: int = 0,
1580        filter_annotated: bool = False,
1581        filter_background: bool = False,
1582        error_class: str = None,
1583        min_frames_action: int = None,
1584        key_objects: Tuple = None,
1585        visibility_min_score: float = 0,
1586        visibility_min_frac: float = 0,
1587        mask: Dict = None,
1588        use_hard_negatives: bool = False,
1589        interactive: bool = False,
1590        ignored_clips: Set = None,
1591        *args,
1592        **kwargs,
1593    ) -> None:
1594        """
1595        Parameters
1596        ----------
1597        video_order : list, optional
1598            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1599        min_frames : dict, optional
1600            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1601            clip start frames (not passed if creating from key objects)
1602        max_frames : dict, optional
1603            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1604            clip end frames (not passed if creating from key objects)
1605        visibility : dict, optional
1606            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1607            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
1608        exclusive : bool, default True
1609            if True, the annotation is single-label; if False, multi-label
1610        len_segment : int, default 128
1611            the length of the segments in which the data should be cut (in frames)
1612        overlap : int, default 0
1613            the length of the overlap between neighboring segments (in frames)
1614        ignored_classes : set, optional
1615            the list of behaviors from the behaviors list or file to not annotate
1616        annotation_path : str | set, optional
1617            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1618            from key objects)
1619        behavior_file : str, optional
1620            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
1621        correction : dict, optional
1622            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
1623            can be used to correct for variations in naming or to merge several labels in one
1624        frame_limit : int, default 0
1625            the smallest possible length of a clip (shorter clips are discarded)
1626        filter_annotated : bool, default False
1627            if True, the samples that do not have any labels will be filtered
1628        filter_background : bool, default False
1629            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
1630        error_class : str, optional
1631            the name of the error class (the annotations that intersect with this label will be discarded)
1632        min_frames_action : int, default 0
1633            the minimum length of an action (shorter actions are not annotated)
1634        key_objects : tuple, optional
1635            the key objects to load the AnnotationStore from
1636        visibility_min_score : float, default 5
1637            the minimum visibility score for visibility filtering
1638        visibility_min_frac : float, default 0.7
1639            the minimum fraction of visible frames for visibility filtering
1640        mask : dict, optional
1641            a masked value dictionary (for active learning simulation experiments)
1642        use_hard_negatives : bool, default False
1643            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
1644        interactive : bool, default False
1645            if `True`, annotation is assigned to pairs of individuals
1646        ignored_clips : set, optional
1647            a set of clip ids to ignore
1648        """
1649
1650        super().__init__(
1651            video_order=video_order,
1652            min_frames=min_frames,
1653            max_frames=max_frames,
1654            visibility=visibility,
1655            exclusive=exclusive,
1656            len_segment=len_segment,
1657            overlap=overlap,
1658            behaviors=None,
1659            ignored_classes=ignored_classes,
1660            annotation_suffix={".txt"},
1661            annotation_path=annotation_path,
1662            behavior_file=behavior_file,
1663            correction=correction,
1664            frame_limit=frame_limit,
1665            filter_annotated=filter_annotated,
1666            filter_background=filter_background,
1667            error_class=error_class,
1668            min_frames_action=min_frames_action,
1669            key_objects=key_objects,
1670            visibility_min_score=visibility_min_score,
1671            visibility_min_frac=visibility_min_frac,
1672            mask=mask,
1673            use_hard_negatives=use_hard_negatives,
1674            interactive=interactive,
1675            ignored_clips=ignored_clips,
1676            *args,
1677            **kwargs,
1678        )
1679
1680    @classmethod
1681    def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List:
1682        """
1683        Process data parameters and return a list of ids  of the videos that should
1684        be processed by the __init__ function
1685
1686        Parameters
1687        ----------
1688        annotation_path : str | set
1689            the path or the set of paths to the folder where the annotation files are stored
1690
1691        Returns
1692        -------
1693        video_ids : list
1694            a list of video file ids
1695        """
1696
1697        if isinstance(annotation_path, str):
1698            annotation_path = [annotation_path]
1699        files = []
1700        for folder in annotation_path:
1701            files += [
1702                os.path.basename(x)[:-4] for x in os.listdir(folder) if x[-4:] == ".txt"
1703            ]
1704        files = sorted(files, key=lambda x: os.path.basename(x))
1705        return files
1706
1707    def _open_annotations(self, filename: str) -> Dict:
1708        """
1709        Load the annotation from filename
1710        """
1711
1712        if self.interactive:
1713            agent_name = "0+1"
1714        else:
1715            agent_name = "0"
1716        times = {agent_name: defaultdict(lambda: [])}
1717        with open(filename) as f:
1718            for line in f.readlines():
1719                label, start, end, *_ = map(int, line.split(","))
1720                times[agent_name][self.all_behaviors[int(label) - 1]].append(
1721                    [start, end, 0]
1722                )
1723        return times
1724
1725    def _set_behaviors(
1726        self, behaviors: List, ignored_classes: List, behavior_file: str
1727    ):
1728        """
1729        Get a list of behaviors that should be annotated from behavior parameters
1730        """
1731
1732        if behavior_file is not None:
1733            behaviors = list(pd.read_excel(behavior_file)["Action"])
1734        self.all_behaviors = copy(behaviors)
1735        for b in ignored_classes:
1736            if b in behaviors:
1737                behaviors.remove(b)
1738        self.behaviors = behaviors

PKU-MMD annotation data

Assumes the following file structure:

annotation_path
├── 0364-L.txt
...
└── 0144-M.txt
PKUMMDAnnotationStore( video_order: List = None, min_frames: Dict = None, max_frames: Dict = None, visibility: Dict = None, exclusive: bool = True, len_segment: int = 128, overlap: int = 0, ignored_classes: Set = None, annotation_path: Union[Set, str] = None, behavior_file: str = None, correction: Dict = None, frame_limit: int = 0, filter_annotated: bool = False, filter_background: bool = False, error_class: str = None, min_frames_action: int = None, key_objects: Tuple = None, visibility_min_score: float = 0, visibility_min_frac: float = 0, mask: Dict = None, use_hard_negatives: bool = False, interactive: bool = False, ignored_clips: Set = None, *args, **kwargs)
1566    def __init__(
1567        self,
1568        video_order: List = None,
1569        min_frames: Dict = None,
1570        max_frames: Dict = None,
1571        visibility: Dict = None,
1572        exclusive: bool = True,
1573        len_segment: int = 128,
1574        overlap: int = 0,
1575        ignored_classes: Set = None,
1576        annotation_path: Union[Set, str] = None,
1577        behavior_file: str = None,
1578        correction: Dict = None,
1579        frame_limit: int = 0,
1580        filter_annotated: bool = False,
1581        filter_background: bool = False,
1582        error_class: str = None,
1583        min_frames_action: int = None,
1584        key_objects: Tuple = None,
1585        visibility_min_score: float = 0,
1586        visibility_min_frac: float = 0,
1587        mask: Dict = None,
1588        use_hard_negatives: bool = False,
1589        interactive: bool = False,
1590        ignored_clips: Set = None,
1591        *args,
1592        **kwargs,
1593    ) -> None:
1594        """
1595        Parameters
1596        ----------
1597        video_order : list, optional
1598            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1599        min_frames : dict, optional
1600            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1601            clip start frames (not passed if creating from key objects)
1602        max_frames : dict, optional
1603            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1604            clip end frames (not passed if creating from key objects)
1605        visibility : dict, optional
1606            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1607            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
1608        exclusive : bool, default True
1609            if True, the annotation is single-label; if False, multi-label
1610        len_segment : int, default 128
1611            the length of the segments in which the data should be cut (in frames)
1612        overlap : int, default 0
1613            the length of the overlap between neighboring segments (in frames)
1614        ignored_classes : set, optional
1615            the list of behaviors from the behaviors list or file to not annotate
1616        annotation_path : str | set, optional
1617            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1618            from key objects)
1619        behavior_file : str, optional
1620            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
1621        correction : dict, optional
1622            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
1623            can be used to correct for variations in naming or to merge several labels in one
1624        frame_limit : int, default 0
1625            the smallest possible length of a clip (shorter clips are discarded)
1626        filter_annotated : bool, default False
1627            if True, the samples that do not have any labels will be filtered
1628        filter_background : bool, default False
1629            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
1630        error_class : str, optional
1631            the name of the error class (the annotations that intersect with this label will be discarded)
1632        min_frames_action : int, default 0
1633            the minimum length of an action (shorter actions are not annotated)
1634        key_objects : tuple, optional
1635            the key objects to load the AnnotationStore from
1636        visibility_min_score : float, default 5
1637            the minimum visibility score for visibility filtering
1638        visibility_min_frac : float, default 0.7
1639            the minimum fraction of visible frames for visibility filtering
1640        mask : dict, optional
1641            a masked value dictionary (for active learning simulation experiments)
1642        use_hard_negatives : bool, default False
1643            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
1644        interactive : bool, default False
1645            if `True`, annotation is assigned to pairs of individuals
1646        ignored_clips : set, optional
1647            a set of clip ids to ignore
1648        """
1649
1650        super().__init__(
1651            video_order=video_order,
1652            min_frames=min_frames,
1653            max_frames=max_frames,
1654            visibility=visibility,
1655            exclusive=exclusive,
1656            len_segment=len_segment,
1657            overlap=overlap,
1658            behaviors=None,
1659            ignored_classes=ignored_classes,
1660            annotation_suffix={".txt"},
1661            annotation_path=annotation_path,
1662            behavior_file=behavior_file,
1663            correction=correction,
1664            frame_limit=frame_limit,
1665            filter_annotated=filter_annotated,
1666            filter_background=filter_background,
1667            error_class=error_class,
1668            min_frames_action=min_frames_action,
1669            key_objects=key_objects,
1670            visibility_min_score=visibility_min_score,
1671            visibility_min_frac=visibility_min_frac,
1672            mask=mask,
1673            use_hard_negatives=use_hard_negatives,
1674            interactive=interactive,
1675            ignored_clips=ignored_clips,
1676            *args,
1677            **kwargs,
1678        )

Parameters

video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one frame_limit : int, default 0 the smallest possible length of a clip (shorter clips are discarded) filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) use_hard_negatives : bool, default False mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing interactive : bool, default False if True, annotation is assigned to pairs of individuals ignored_clips : set, optional a set of clip ids to ignore

@classmethod
def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List:
1680    @classmethod
1681    def get_file_ids(cls, annotation_path: Union[str, Set], *args, **kwargs) -> List:
1682        """
1683        Process data parameters and return a list of ids  of the videos that should
1684        be processed by the __init__ function
1685
1686        Parameters
1687        ----------
1688        annotation_path : str | set
1689            the path or the set of paths to the folder where the annotation files are stored
1690
1691        Returns
1692        -------
1693        video_ids : list
1694            a list of video file ids
1695        """
1696
1697        if isinstance(annotation_path, str):
1698            annotation_path = [annotation_path]
1699        files = []
1700        for folder in annotation_path:
1701            files += [
1702                os.path.basename(x)[:-4] for x in os.listdir(folder) if x[-4:] == ".txt"
1703            ]
1704        files = sorted(files, key=lambda x: os.path.basename(x))
1705        return files

Process data parameters and return a list of ids of the videos that should be processed by the __init__ function

Parameters

annotation_path : str | set the path or the set of paths to the folder where the annotation files are stored

Returns

video_ids : list a list of video file ids

class CalMS21AnnotationStore(SequenceAnnotationStore):
1741class CalMS21AnnotationStore(SequenceAnnotationStore):  # +
1742    """
1743    CalMS21 annotation data
1744
1745    Use the `'random:test_from_name:{name}'` and `'val-from-name:{val_name}:test-from-name:{test_name}'`
1746    partitioning methods with `'train'`, `'test'` and `'unlabeled'` names to separate into train, test and validation
1747    subsets according to the original files. For example, with `'val-from-name:test:test-from-name:unlabeled'`
1748    the data from the test file will go into validation and the unlabeled files will be the test.
1749
1750    Assumes the following file structure:
1751    ```
1752    annotation_path
1753    ├── calms21_task_train.npy
1754    ├── calms21_task_test.npy
1755    ├── calms21_unlabeled_videos_part1.npy
1756    ├── calms21_unlabeled_videos_part2.npy
1757    └── calms21_unlabeled_videos_part3.npy
1758    ```
1759    """
1760
1761    def __init__(
1762        self,
1763        task_n: int = 1,
1764        include_task1: bool = True,
1765        video_order: List = None,
1766        min_frames: Dict = None,
1767        max_frames: Dict = None,
1768        len_segment: int = 128,
1769        overlap: int = 0,
1770        ignored_classes: Set = None,
1771        annotation_path: Union[Set, str] = None,
1772        key_objects: Tuple = None,
1773        treba_files: bool = False,
1774        *args,
1775        **kwargs,
1776    ) -> None:
1777        """
1778
1779        Parameters
1780        ----------
1781        task_n : [1, 2]
1782            the number of the task
1783        include_task1 : bool, default True
1784            include task 1 data to training set
1785        video_order : list, optional
1786            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1787        min_frames : dict, optional
1788            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1789            clip start frames (not passed if creating from key objects)
1790        max_frames : dict, optional
1791            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1792            clip end frames (not passed if creating from key objects)
1793        len_segment : int, default 128
1794            the length of the segments in which the data should be cut (in frames)
1795        overlap : int, default 0
1796            the length of the overlap between neighboring segments (in frames)
1797        ignored_classes : set, optional
1798            the list of behaviors from the behaviors list or file to not annotate
1799        annotation_path : str | set, optional
1800            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1801            from key objects)
1802        key_objects : tuple, optional
1803            the key objects to load the AnnotationStore from
1804        treba_files : bool, default False
1805            if `True`, TREBA feature files will be loaded
1806        """
1807
1808        self.task_n = int(task_n)
1809        self.include_task1 = include_task1
1810        if self.task_n == 1:
1811            self.include_task1 = False
1812        self.treba_files = treba_files
1813        if "exclusive" in kwargs:
1814            exclusive = kwargs["exclusive"]
1815        else:
1816            exclusive = True
1817        if "behaviors" in kwargs and kwargs["behaviors"] is not None:
1818            behaviors = kwargs["behaviors"]
1819        else:
1820            behaviors = ["attack", "investigation", "mount", "other"]
1821            if task_n == 3:
1822                exclusive = False
1823                behaviors += [
1824                    "approach",
1825                    "disengaged",
1826                    "groom",
1827                    "intromission",
1828                    "mount_attempt",
1829                    "sniff_face",
1830                    "whiterearing",
1831                ]
1832        super().__init__(
1833            video_order=video_order,
1834            min_frames=min_frames,
1835            max_frames=max_frames,
1836            exclusive=exclusive,
1837            len_segment=len_segment,
1838            overlap=overlap,
1839            behaviors=behaviors,
1840            ignored_classes=ignored_classes,
1841            annotation_path=annotation_path,
1842            key_objects=key_objects,
1843            filter_annotated=False,
1844            interactive=True,
1845        )
1846
1847    @classmethod
1848    def get_file_ids(
1849        cls,
1850        task_n: int = 1,
1851        include_task1: bool = False,
1852        treba_files: bool = False,
1853        annotation_path: Union[str, Set] = None,
1854        file_paths=None,
1855        *args,
1856        **kwargs,
1857    ) -> Iterable:
1858        """
1859        Process data parameters and return a list of ids  of the videos that should
1860        be processed by the __init__ function
1861
1862        Parameters
1863        ----------
1864        task_n : {1, 2, 3}
1865            the index of the CalMS21 challenge task
1866        include_task1 : bool, default False
1867            if `True`, the training file of the task 1 will be loaded
1868        treba_files : bool, default False
1869            if `True`, the TREBA feature files will be loaded
1870        filenames : set, optional
1871            a set of string filenames to search for (only basenames, not the whole paths)
1872        annotation_path : str | set, optional
1873            the path to the folder where the pose and feature files are stored or a set of such paths
1874            (not passed if creating from key objects or from `file_paths`)
1875        file_paths : set, optional
1876            a set of string paths to the pose and feature files
1877            (not passed if creating from key objects or from `data_path`)
1878
1879        Returns
1880        -------
1881        video_ids : list
1882            a list of video file ids
1883        """
1884
1885        task_n = int(task_n)
1886        if task_n == 1:
1887            include_task1 = False
1888        files = []
1889        if treba_files:
1890            postfix = "_features"
1891        else:
1892            postfix = ""
1893        files.append(f"calms21_task{task_n}_train{postfix}.npy")
1894        files.append(f"calms21_task{task_n}_test{postfix}.npy")
1895        if include_task1:
1896            files.append(f"calms21_task1_train{postfix}.npy")
1897        filenames = set(files)
1898        return SequenceAnnotationStore.get_file_ids(
1899            filenames, annotation_path=annotation_path
1900        )
1901
1902    def _open_sequences(self, filename: str) -> Dict:
1903        """
1904        Load the annotation from filename
1905
1906        Parameters
1907        ----------
1908        filename : str
1909            path to an annotation file
1910
1911        Returns
1912        -------
1913        times : dict
1914            a nested dictionary where first-level keys are video ids, second-level keys are clip ids,
1915            third-level keys are categories and values are
1916            lists of (start, end, ambiguity status) lists
1917        """
1918
1919        data_dict = np.load(filename, allow_pickle=True).item()
1920        data = {}
1921        result = {}
1922        keys = list(data_dict.keys())
1923        if "test" in os.path.basename(filename):
1924            mode = "test"
1925        elif "unlabeled" in os.path.basename(filename):
1926            mode = "unlabeled"
1927        else:
1928            mode = "train"
1929        if "approach" in keys:
1930            for behavior in keys:
1931                for key in data_dict[behavior].keys():
1932                    ann = data_dict[behavior][key]["annotations"]
1933                    result[f'{mode}--{key.split("/")[-1]}'] = {
1934                        "mouse1+mouse2": defaultdict(lambda: [])
1935                    }
1936                    starts = np.where(
1937                        np.diff(np.concatenate([np.array([0]), ann, np.array([0])]))
1938                        == 1
1939                    )[0]
1940                    ends = np.where(
1941                        np.diff(np.concatenate([np.array([0]), ann, np.array([0])]))
1942                        == -1
1943                    )[0]
1944                    for start, end in zip(starts, ends):
1945                        result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][
1946                            behavior
1947                        ].append([start, end, 0])
1948                    for b in self.behaviors:
1949                        result[f'{mode}--{key.split("/")[-1]}---mouse1+mouse2'][
1950                            "mouse1+mouse2"
1951                        ][f"unknown {b}"].append([0, len(ann), 0])
1952        for key in keys:
1953            data.update(data_dict[key])
1954            data_dict.pop(key)
1955        if "approach" not in keys and self.task_n == 3:
1956            for key in data.keys():
1957                result[f'{mode}--{key.split("/")[-1]}'] = {"mouse1+mouse2": {}}
1958                ann = data[key]["annotations"]
1959                for i in range(4):
1960                    starts = np.where(
1961                        np.diff(
1962                            np.concatenate(
1963                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1964                            )
1965                        )
1966                        == 1
1967                    )[0]
1968                    ends = np.where(
1969                        np.diff(
1970                            np.concatenate(
1971                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1972                            )
1973                        )
1974                        == -1
1975                    )[0]
1976                    result[f'{mode}--{key.split("/")[-1]}']["mouse1+mouse2"][
1977                        self.behaviors_dict()[i]
1978                    ] = [[start, end, 0] for start, end in zip(starts, ends)]
1979        if self.task_n != 3:
1980            for seq_name, seq_dict in data.items():
1981                if "annotations" not in seq_dict:
1982                    return None
1983                behaviors = np.unique(seq_dict["annotations"])
1984                ann = seq_dict["annotations"]
1985                key = f'{mode}--{seq_name.split("/")[-1]}'
1986                result[key] = {"mouse1+mouse2": {}}
1987                for i in behaviors:
1988                    starts = np.where(
1989                        np.diff(
1990                            np.concatenate(
1991                                [np.array([0]), (ann == i).astype(int), np.array([0])]
1992                            )
1993                        )
1994                        == 1
1995                    )[0]
1996                    ends = np.where(
1997                        np.diff(
1998                            np.concatenate(
1999                                [np.array([0]), (ann == i).astype(int), np.array([0])]
2000                            )
2001                        )
2002                        == -1
2003                    )[0]
2004                    result[key]["mouse1+mouse2"][self.behaviors_dict()[i]] = [
2005                        [start, end, 0] for start, end in zip(starts, ends)
2006                    ]
2007        return result

CalMS21 annotation data

Use the 'random:test_from_name:{name}' and 'val-from-name:{val_name}:test-from-name:{test_name}' partitioning methods with 'train', 'test' and 'unlabeled' names to separate into train, test and validation subsets according to the original files. For example, with 'val-from-name:test:test-from-name:unlabeled' the data from the test file will go into validation and the unlabeled files will be the test.

Assumes the following file structure:

annotation_path
├── calms21_task_train.npy
├── calms21_task_test.npy
├── calms21_unlabeled_videos_part1.npy
├── calms21_unlabeled_videos_part2.npy
└── calms21_unlabeled_videos_part3.npy
CalMS21AnnotationStore( task_n: int = 1, include_task1: bool = True, video_order: List = None, min_frames: Dict = None, max_frames: Dict = None, len_segment: int = 128, overlap: int = 0, ignored_classes: Set = None, annotation_path: Union[Set, str] = None, key_objects: Tuple = None, treba_files: bool = False, *args, **kwargs)
1761    def __init__(
1762        self,
1763        task_n: int = 1,
1764        include_task1: bool = True,
1765        video_order: List = None,
1766        min_frames: Dict = None,
1767        max_frames: Dict = None,
1768        len_segment: int = 128,
1769        overlap: int = 0,
1770        ignored_classes: Set = None,
1771        annotation_path: Union[Set, str] = None,
1772        key_objects: Tuple = None,
1773        treba_files: bool = False,
1774        *args,
1775        **kwargs,
1776    ) -> None:
1777        """
1778
1779        Parameters
1780        ----------
1781        task_n : [1, 2]
1782            the number of the task
1783        include_task1 : bool, default True
1784            include task 1 data to training set
1785        video_order : list, optional
1786            a list of video ids that should be processed in the same order (not passed if creating from key objects)
1787        min_frames : dict, optional
1788            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1789            clip start frames (not passed if creating from key objects)
1790        max_frames : dict, optional
1791            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
1792            clip end frames (not passed if creating from key objects)
1793        len_segment : int, default 128
1794            the length of the segments in which the data should be cut (in frames)
1795        overlap : int, default 0
1796            the length of the overlap between neighboring segments (in frames)
1797        ignored_classes : set, optional
1798            the list of behaviors from the behaviors list or file to not annotate
1799        annotation_path : str | set, optional
1800            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
1801            from key objects)
1802        key_objects : tuple, optional
1803            the key objects to load the AnnotationStore from
1804        treba_files : bool, default False
1805            if `True`, TREBA feature files will be loaded
1806        """
1807
1808        self.task_n = int(task_n)
1809        self.include_task1 = include_task1
1810        if self.task_n == 1:
1811            self.include_task1 = False
1812        self.treba_files = treba_files
1813        if "exclusive" in kwargs:
1814            exclusive = kwargs["exclusive"]
1815        else:
1816            exclusive = True
1817        if "behaviors" in kwargs and kwargs["behaviors"] is not None:
1818            behaviors = kwargs["behaviors"]
1819        else:
1820            behaviors = ["attack", "investigation", "mount", "other"]
1821            if task_n == 3:
1822                exclusive = False
1823                behaviors += [
1824                    "approach",
1825                    "disengaged",
1826                    "groom",
1827                    "intromission",
1828                    "mount_attempt",
1829                    "sniff_face",
1830                    "whiterearing",
1831                ]
1832        super().__init__(
1833            video_order=video_order,
1834            min_frames=min_frames,
1835            max_frames=max_frames,
1836            exclusive=exclusive,
1837            len_segment=len_segment,
1838            overlap=overlap,
1839            behaviors=behaviors,
1840            ignored_classes=ignored_classes,
1841            annotation_path=annotation_path,
1842            key_objects=key_objects,
1843            filter_annotated=False,
1844            interactive=True,
1845        )

Parameters

task_n : [1, 2] the number of the task include_task1 : bool, default True include task 1 data to training set video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) key_objects : tuple, optional the key objects to load the AnnotationStore from treba_files : bool, default False if True, TREBA feature files will be loaded

@classmethod
def get_file_ids( cls, task_n: int = 1, include_task1: bool = False, treba_files: bool = False, annotation_path: Union[str, Set] = None, file_paths=None, *args, **kwargs) -> collections.abc.Iterable:
1847    @classmethod
1848    def get_file_ids(
1849        cls,
1850        task_n: int = 1,
1851        include_task1: bool = False,
1852        treba_files: bool = False,
1853        annotation_path: Union[str, Set] = None,
1854        file_paths=None,
1855        *args,
1856        **kwargs,
1857    ) -> Iterable:
1858        """
1859        Process data parameters and return a list of ids  of the videos that should
1860        be processed by the __init__ function
1861
1862        Parameters
1863        ----------
1864        task_n : {1, 2, 3}
1865            the index of the CalMS21 challenge task
1866        include_task1 : bool, default False
1867            if `True`, the training file of the task 1 will be loaded
1868        treba_files : bool, default False
1869            if `True`, the TREBA feature files will be loaded
1870        filenames : set, optional
1871            a set of string filenames to search for (only basenames, not the whole paths)
1872        annotation_path : str | set, optional
1873            the path to the folder where the pose and feature files are stored or a set of such paths
1874            (not passed if creating from key objects or from `file_paths`)
1875        file_paths : set, optional
1876            a set of string paths to the pose and feature files
1877            (not passed if creating from key objects or from `data_path`)
1878
1879        Returns
1880        -------
1881        video_ids : list
1882            a list of video file ids
1883        """
1884
1885        task_n = int(task_n)
1886        if task_n == 1:
1887            include_task1 = False
1888        files = []
1889        if treba_files:
1890            postfix = "_features"
1891        else:
1892            postfix = ""
1893        files.append(f"calms21_task{task_n}_train{postfix}.npy")
1894        files.append(f"calms21_task{task_n}_test{postfix}.npy")
1895        if include_task1:
1896            files.append(f"calms21_task1_train{postfix}.npy")
1897        filenames = set(files)
1898        return SequenceAnnotationStore.get_file_ids(
1899            filenames, annotation_path=annotation_path
1900        )

Process data parameters and return a list of ids of the videos that should be processed by the __init__ function

Parameters

task_n : {1, 2, 3} the index of the CalMS21 challenge task include_task1 : bool, default False if True, the training file of the task 1 will be loaded treba_files : bool, default False if True, the TREBA feature files will be loaded filenames : set, optional a set of string filenames to search for (only basenames, not the whole paths) annotation_path : str | set, optional the path to the folder where the pose and feature files are stored or a set of such paths (not passed if creating from key objects or from file_paths) file_paths : set, optional a set of string paths to the pose and feature files (not passed if creating from key objects or from data_path)

Returns

video_ids : list a list of video file ids

class CSVAnnotationStore(FileAnnotationStore):
2010class CSVAnnotationStore(FileAnnotationStore):  # +
2011    """
2012    CSV type annotation data
2013
2014    Assumes that files are saved as .csv tables with at least the following columns:
2015    - from / start : start of action,
2016    - to / end : end of action,
2017    - class / behavior / behaviour / label / type : action label.
2018
2019    If the times are set in seconds instead of frames, don't forget to set the `fps` parameter to your frame rate.
2020
2021    Assumes the following file structure:
2022    ```
2023    annotation_path
2024    ├── video1_annotation.csv
2025    └── video2_labels.csv
2026    ```
2027    Here `annotation_suffix` is `{'_annotation.csv', '_labels.csv'}`.
2028    """
2029
2030    def __init__(
2031        self,
2032        video_order: List = None,
2033        min_frames: Dict = None,
2034        max_frames: Dict = None,
2035        visibility: Dict = None,
2036        exclusive: bool = True,
2037        len_segment: int = 128,
2038        overlap: int = 0,
2039        behaviors: Set = None,
2040        ignored_classes: Set = None,
2041        annotation_suffix: Union[Set, str] = None,
2042        annotation_path: Union[Set, str] = None,
2043        behavior_file: str = None,
2044        correction: Dict = None,
2045        frame_limit: int = 0,
2046        filter_annotated: bool = False,
2047        filter_background: bool = False,
2048        error_class: str = None,
2049        min_frames_action: int = None,
2050        key_objects: Tuple = None,
2051        visibility_min_score: float = 0.2,
2052        visibility_min_frac: float = 0.7,
2053        mask: Dict = None,
2054        default_agent_name: str = "ind0",
2055        separator: str = ",",
2056        fps: int = 30,
2057        *args,
2058        **kwargs,
2059    ) -> None:
2060        """
2061        Parameters
2062        ----------
2063        video_order : list, optional
2064            a list of video ids that should be processed in the same order (not passed if creating from key objects)
2065        min_frames : dict, optional
2066            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2067            clip start frames (not passed if creating from key objects)
2068        max_frames : dict, optional
2069            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2070            clip end frames (not passed if creating from key objects)
2071        visibility : dict, optional
2072            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2073            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
2074        exclusive : bool, default True
2075            if True, the annotation is single-label; if False, multi-label
2076        len_segment : int, default 128
2077            the length of the segments in which the data should be cut (in frames)
2078        overlap : int, default 0
2079            the length of the overlap between neighboring segments (in frames)
2080        behaviors : set, optional
2081            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
2082            loaded from a file)
2083        ignored_classes : set, optional
2084            the list of behaviors from the behaviors list or file to not annotate
2085        annotation_suffix : str | set, optional
2086            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
2087            (not passed if creating from key objects or if irrelevant for the dataset)
2088        annotation_path : str | set, optional
2089            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
2090            from key objects)
2091        behavior_file : str, optional
2092            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
2093        correction : dict, optional
2094            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
2095            can be used to correct for variations in naming or to merge several labels in one
2096        frame_limit : int, default 0
2097            the smallest possible length of a clip (shorter clips are discarded)
2098        filter_annotated : bool, default False
2099            if True, the samples that do not have any labels will be filtered
2100        filter_background : bool, default False
2101            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
2102        error_class : str, optional
2103            the name of the error class (the annotations that intersect with this label will be discarded)
2104        min_frames_action : int, default 0
2105            the minimum length of an action (shorter actions are not annotated)
2106        key_objects : tuple, optional
2107            the key objects to load the AnnotationStore from
2108        visibility_min_score : float, default 5
2109            the minimum visibility score for visibility filtering
2110        visibility_min_frac : float, default 0.7
2111            the minimum fraction of visible frames for visibility filtering
2112        mask : dict, optional
2113            a masked value dictionary (for active learning simulation experiments)
2114        default_agent_name : str, default "ind0"
2115            the clip id to use when there is no given
2116        separator : str, default ","
2117            the separator in the csv files
2118        fps : int, default 30
2119            frames per second in the videos
2120        """
2121
2122        self.default_agent_name = default_agent_name
2123        self.separator = separator
2124        self.fps = fps
2125        super().__init__(
2126            video_order=video_order,
2127            min_frames=min_frames,
2128            max_frames=max_frames,
2129            visibility=visibility,
2130            exclusive=exclusive,
2131            len_segment=len_segment,
2132            overlap=overlap,
2133            behaviors=behaviors,
2134            ignored_classes=ignored_classes,
2135            ignored_clips=None,
2136            annotation_suffix=annotation_suffix,
2137            annotation_path=annotation_path,
2138            behavior_file=behavior_file,
2139            correction=correction,
2140            frame_limit=frame_limit,
2141            filter_annotated=filter_annotated,
2142            filter_background=filter_background,
2143            error_class=error_class,
2144            min_frames_action=min_frames_action,
2145            key_objects=key_objects,
2146            visibility_min_score=visibility_min_score,
2147            visibility_min_frac=visibility_min_frac,
2148            mask=mask,
2149        )
2150
2151    def _open_annotations(self, filename: str) -> Dict:
2152        """
2153        Load the annotation from `filename`
2154        """
2155
2156        # try:
2157        data = pd.read_csv(filename, sep=self.separator)
2158        data.columns = list(map(lambda x: x.lower(), data.columns))
2159        starts, ends, actions = None, None, None
2160        start_names = ["from", "start"]
2161        for x in start_names:
2162            if x in data.columns:
2163                starts = data[x]
2164        end_names = ["to", "end"]
2165        for x in end_names:
2166            if x in data.columns:
2167                ends = data[x]
2168        class_names = ["class", "behavior", "behaviour", "type", "label"]
2169        for x in class_names:
2170            if x in data.columns:
2171                actions = data[x]
2172        if starts is None:
2173            raise ValueError("The file must have a column titled 'from' or 'start'!")
2174        if ends is None:
2175            raise ValueError("The file must have a column titled 'to' or 'end'!")
2176        if actions is None:
2177            raise ValueError(
2178                "The file must have a column titled 'class', 'behavior', 'behaviour', 'type' or 'label'!"
2179            )
2180        times = defaultdict(lambda: defaultdict(lambda: []))
2181        for start, end, action in zip(starts, ends, actions):
2182            if any([np.isnan(x) for x in [start, end]]):
2183                continue
2184            times[self.default_agent_name][action].append(
2185                [int(start * self.fps), int(end * self.fps), 0]
2186            )
2187        return times

CSV type annotation data

Assumes that files are saved as .csv tables with at least the following columns:

  • from / start : start of action,
  • to / end : end of action,
  • class / behavior / behaviour / label / type : action label.

If the times are set in seconds instead of frames, don't forget to set the fps parameter to your frame rate.

Assumes the following file structure:

annotation_path
├── video1_annotation.csv
└── video2_labels.csv

Here annotation_suffix is {'_annotation.csv', '_labels.csv'}.

CSVAnnotationStore( video_order: List = None, min_frames: Dict = None, max_frames: Dict = None, visibility: Dict = None, exclusive: bool = True, len_segment: int = 128, overlap: int = 0, behaviors: Set = None, ignored_classes: Set = None, annotation_suffix: Union[Set, str] = None, annotation_path: Union[Set, str] = None, behavior_file: str = None, correction: Dict = None, frame_limit: int = 0, filter_annotated: bool = False, filter_background: bool = False, error_class: str = None, min_frames_action: int = None, key_objects: Tuple = None, visibility_min_score: float = 0.2, visibility_min_frac: float = 0.7, mask: Dict = None, default_agent_name: str = 'ind0', separator: str = ',', fps: int = 30, *args, **kwargs)
2030    def __init__(
2031        self,
2032        video_order: List = None,
2033        min_frames: Dict = None,
2034        max_frames: Dict = None,
2035        visibility: Dict = None,
2036        exclusive: bool = True,
2037        len_segment: int = 128,
2038        overlap: int = 0,
2039        behaviors: Set = None,
2040        ignored_classes: Set = None,
2041        annotation_suffix: Union[Set, str] = None,
2042        annotation_path: Union[Set, str] = None,
2043        behavior_file: str = None,
2044        correction: Dict = None,
2045        frame_limit: int = 0,
2046        filter_annotated: bool = False,
2047        filter_background: bool = False,
2048        error_class: str = None,
2049        min_frames_action: int = None,
2050        key_objects: Tuple = None,
2051        visibility_min_score: float = 0.2,
2052        visibility_min_frac: float = 0.7,
2053        mask: Dict = None,
2054        default_agent_name: str = "ind0",
2055        separator: str = ",",
2056        fps: int = 30,
2057        *args,
2058        **kwargs,
2059    ) -> None:
2060        """
2061        Parameters
2062        ----------
2063        video_order : list, optional
2064            a list of video ids that should be processed in the same order (not passed if creating from key objects)
2065        min_frames : dict, optional
2066            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2067            clip start frames (not passed if creating from key objects)
2068        max_frames : dict, optional
2069            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2070            clip end frames (not passed if creating from key objects)
2071        visibility : dict, optional
2072            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2073            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
2074        exclusive : bool, default True
2075            if True, the annotation is single-label; if False, multi-label
2076        len_segment : int, default 128
2077            the length of the segments in which the data should be cut (in frames)
2078        overlap : int, default 0
2079            the length of the overlap between neighboring segments (in frames)
2080        behaviors : set, optional
2081            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
2082            loaded from a file)
2083        ignored_classes : set, optional
2084            the list of behaviors from the behaviors list or file to not annotate
2085        annotation_suffix : str | set, optional
2086            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
2087            (not passed if creating from key objects or if irrelevant for the dataset)
2088        annotation_path : str | set, optional
2089            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
2090            from key objects)
2091        behavior_file : str, optional
2092            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
2093        correction : dict, optional
2094            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
2095            can be used to correct for variations in naming or to merge several labels in one
2096        frame_limit : int, default 0
2097            the smallest possible length of a clip (shorter clips are discarded)
2098        filter_annotated : bool, default False
2099            if True, the samples that do not have any labels will be filtered
2100        filter_background : bool, default False
2101            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
2102        error_class : str, optional
2103            the name of the error class (the annotations that intersect with this label will be discarded)
2104        min_frames_action : int, default 0
2105            the minimum length of an action (shorter actions are not annotated)
2106        key_objects : tuple, optional
2107            the key objects to load the AnnotationStore from
2108        visibility_min_score : float, default 5
2109            the minimum visibility score for visibility filtering
2110        visibility_min_frac : float, default 0.7
2111            the minimum fraction of visible frames for visibility filtering
2112        mask : dict, optional
2113            a masked value dictionary (for active learning simulation experiments)
2114        default_agent_name : str, default "ind0"
2115            the clip id to use when there is no given
2116        separator : str, default ","
2117            the separator in the csv files
2118        fps : int, default 30
2119            frames per second in the videos
2120        """
2121
2122        self.default_agent_name = default_agent_name
2123        self.separator = separator
2124        self.fps = fps
2125        super().__init__(
2126            video_order=video_order,
2127            min_frames=min_frames,
2128            max_frames=max_frames,
2129            visibility=visibility,
2130            exclusive=exclusive,
2131            len_segment=len_segment,
2132            overlap=overlap,
2133            behaviors=behaviors,
2134            ignored_classes=ignored_classes,
2135            ignored_clips=None,
2136            annotation_suffix=annotation_suffix,
2137            annotation_path=annotation_path,
2138            behavior_file=behavior_file,
2139            correction=correction,
2140            frame_limit=frame_limit,
2141            filter_annotated=filter_annotated,
2142            filter_background=filter_background,
2143            error_class=error_class,
2144            min_frames_action=min_frames_action,
2145            key_objects=key_objects,
2146            visibility_min_score=visibility_min_score,
2147            visibility_min_frac=visibility_min_frac,
2148            mask=mask,
2149        )

Parameters

video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) behaviors : set, optional the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are loaded from a file) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} (not passed if creating from key objects or if irrelevant for the dataset) annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one frame_limit : int, default 0 the smallest possible length of a clip (shorter clips are discarded) filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) default_agent_name : str, default "ind0" the clip id to use when there is no given separator : str, default "," the separator in the csv files fps : int, default 30 frames per second in the videos

class SIMBAAnnotationStore(FileAnnotationStore):
2190class SIMBAAnnotationStore(FileAnnotationStore):  # +
2191    """
2192    SIMBA paper format data
2193
2194    Assumes the following file structure:
2195    ```
2196    annotation_path
2197    ├── Video1.csv
2198    ...
2199    └── Video9.csv
2200    """
2201
2202    def __init__(
2203        self,
2204        video_order: List = None,
2205        min_frames: Dict = None,
2206        max_frames: Dict = None,
2207        visibility: Dict = None,
2208        exclusive: bool = True,
2209        len_segment: int = 128,
2210        overlap: int = 0,
2211        behaviors: Set = None,
2212        ignored_classes: Set = None,
2213        ignored_clips: Set = None,
2214        annotation_path: Union[Set, str] = None,
2215        correction: Dict = None,
2216        filter_annotated: bool = False,
2217        filter_background: bool = False,
2218        error_class: str = None,
2219        min_frames_action: int = None,
2220        key_objects: Tuple = None,
2221        visibility_min_score: float = 0.2,
2222        visibility_min_frac: float = 0.7,
2223        mask: Dict = None,
2224        use_hard_negatives: bool = False,
2225        annotation_suffix: str = None,
2226        *args,
2227        **kwargs,
2228    ) -> None:
2229        """
2230        Parameters
2231        ----------
2232        video_order : list, optional
2233            a list of video ids that should be processed in the same order (not passed if creating from key objects)
2234        min_frames : dict, optional
2235            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2236            clip start frames (not passed if creating from key objects)
2237        max_frames : dict, optional
2238            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2239            clip end frames (not passed if creating from key objects)
2240        visibility : dict, optional
2241            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2242            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
2243        exclusive : bool, default True
2244            if True, the annotation is single-label; if False, multi-label
2245        len_segment : int, default 128
2246            the length of the segments in which the data should be cut (in frames)
2247        overlap : int, default 0
2248            the length of the overlap between neighboring segments (in frames)
2249        behaviors : set, optional
2250            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
2251            loaded from a file)
2252        ignored_classes : set, optional
2253            the list of behaviors from the behaviors list or file to not annotate
2254        ignored_clips : set, optional
2255            clip ids to ignore
2256        annotation_path : str | set, optional
2257            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
2258            from key objects)
2259        behavior_file : str, optional
2260            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
2261        correction : dict, optional
2262            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
2263            can be used to correct for variations in naming or to merge several labels in one
2264        filter_annotated : bool, default False
2265            if True, the samples that do not have any labels will be filtered
2266        filter_background : bool, default False
2267            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
2268        error_class : str, optional
2269            the name of the error class (the annotations that intersect with this label will be discarded)
2270        min_frames_action : int, default 0
2271            the minimum length of an action (shorter actions are not annotated)
2272        key_objects : tuple, optional
2273            the key objects to load the AnnotationStore from
2274        visibility_min_score : float, default 5
2275            the minimum visibility score for visibility filtering
2276        visibility_min_frac : float, default 0.7
2277            the minimum fraction of visible frames for visibility filtering
2278        mask : dict, optional
2279            a masked value dictionary (for active learning simulation experiments)
2280        use_hard_negatives : bool, default False
2281            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
2282        annotation_suffix : str | set, optional
2283            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
2284            (not passed if creating from key objects or if irrelevant for the dataset)
2285        """
2286
2287        super().__init__(
2288            video_order=video_order,
2289            min_frames=min_frames,
2290            max_frames=max_frames,
2291            visibility=visibility,
2292            exclusive=exclusive,
2293            len_segment=len_segment,
2294            overlap=overlap,
2295            behaviors=behaviors,
2296            ignored_classes=ignored_classes,
2297            ignored_clips=ignored_clips,
2298            annotation_suffix=annotation_suffix,
2299            annotation_path=annotation_path,
2300            behavior_file=None,
2301            correction=correction,
2302            frame_limit=0,
2303            filter_annotated=filter_annotated,
2304            filter_background=filter_background,
2305            error_class=error_class,
2306            min_frames_action=min_frames_action,
2307            key_objects=key_objects,
2308            visibility_min_score=visibility_min_score,
2309            visibility_min_frac=visibility_min_frac,
2310            mask=mask,
2311            use_hard_negatives=use_hard_negatives,
2312            interactive=True,
2313        )
2314
2315    def _open_annotations(self, filename: str) -> Dict:
2316        """
2317        Load the annotation from filename
2318
2319        Parameters
2320        ----------
2321        filename : str
2322            path to an annotation file
2323
2324        Returns
2325        -------
2326        times : dict
2327            a nested dictionary where first-level keys are clip ids, second-level keys are categories and values are
2328            lists of (start, end, ambiguity status) lists
2329        """
2330
2331        data = pd.read_csv(filename)
2332        columns = [x for x in data.columns if x.split("_")[-1] == "x"]
2333        animals = sorted(set([x.split("_")[-2] for x in columns]))
2334        if len(animals) > 2:
2335            raise ValueError(
2336                "SIMBAAnnotationStore is only implemented for files with 1 or 2 animals!"
2337            )
2338        if len(animals) == 1:
2339            ind = animals[0]
2340        else:
2341            ind = "+".join(animals)
2342        behaviors = [
2343            "_".join(x.split("_")[:-1])
2344            for x in data.columns
2345            if x.split("_")[-1] == "prediction"
2346        ]
2347        result = {}
2348        for behavior in behaviors:
2349            ann = data[f"{behavior}_prediction"].values
2350            diff = np.diff(
2351                np.concatenate([np.array([0]), (ann == 1).astype(int), np.array([0])])
2352            )
2353            starts = np.where(diff == 1)[0]
2354            ends = np.where(diff == -1)[0]
2355            result[behavior] = [[start, end, 0] for start, end in zip(starts, ends)]
2356            diff = np.diff(
2357                np.concatenate(
2358                    [np.array([0]), (np.isnan(ann)).astype(int), np.array([0])]
2359                )
2360            )
2361            starts = np.where(diff == 1)[0]
2362            ends = np.where(diff == -1)[0]
2363            result[f"unknown {behavior}"] = [
2364                [start, end, 0] for start, end in zip(starts, ends)
2365            ]
2366        if self.behaviors is not None:
2367            for behavior in self.behaviors:
2368                if behavior not in behaviors:
2369                    result[f"unknown {behavior}"] = [[0, len(data), 0]]
2370        return {ind: result}

SIMBA paper format data

Assumes the following file structure: ``` annotation_path ├── Video1.csv ... └── Video9.csv

SIMBAAnnotationStore( video_order: List = None, min_frames: Dict = None, max_frames: Dict = None, visibility: Dict = None, exclusive: bool = True, len_segment: int = 128, overlap: int = 0, behaviors: Set = None, ignored_classes: Set = None, ignored_clips: Set = None, annotation_path: Union[Set, str] = None, correction: Dict = None, filter_annotated: bool = False, filter_background: bool = False, error_class: str = None, min_frames_action: int = None, key_objects: Tuple = None, visibility_min_score: float = 0.2, visibility_min_frac: float = 0.7, mask: Dict = None, use_hard_negatives: bool = False, annotation_suffix: str = None, *args, **kwargs)
2202    def __init__(
2203        self,
2204        video_order: List = None,
2205        min_frames: Dict = None,
2206        max_frames: Dict = None,
2207        visibility: Dict = None,
2208        exclusive: bool = True,
2209        len_segment: int = 128,
2210        overlap: int = 0,
2211        behaviors: Set = None,
2212        ignored_classes: Set = None,
2213        ignored_clips: Set = None,
2214        annotation_path: Union[Set, str] = None,
2215        correction: Dict = None,
2216        filter_annotated: bool = False,
2217        filter_background: bool = False,
2218        error_class: str = None,
2219        min_frames_action: int = None,
2220        key_objects: Tuple = None,
2221        visibility_min_score: float = 0.2,
2222        visibility_min_frac: float = 0.7,
2223        mask: Dict = None,
2224        use_hard_negatives: bool = False,
2225        annotation_suffix: str = None,
2226        *args,
2227        **kwargs,
2228    ) -> None:
2229        """
2230        Parameters
2231        ----------
2232        video_order : list, optional
2233            a list of video ids that should be processed in the same order (not passed if creating from key objects)
2234        min_frames : dict, optional
2235            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2236            clip start frames (not passed if creating from key objects)
2237        max_frames : dict, optional
2238            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2239            clip end frames (not passed if creating from key objects)
2240        visibility : dict, optional
2241            a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are
2242            visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset)
2243        exclusive : bool, default True
2244            if True, the annotation is single-label; if False, multi-label
2245        len_segment : int, default 128
2246            the length of the segments in which the data should be cut (in frames)
2247        overlap : int, default 0
2248            the length of the overlap between neighboring segments (in frames)
2249        behaviors : set, optional
2250            the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are
2251            loaded from a file)
2252        ignored_classes : set, optional
2253            the list of behaviors from the behaviors list or file to not annotate
2254        ignored_clips : set, optional
2255            clip ids to ignore
2256        annotation_path : str | set, optional
2257            the path or the set of paths to the folder where the annotation files are stored (not passed if creating
2258            from key objects)
2259        behavior_file : str, optional
2260            the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset)
2261        correction : dict, optional
2262            a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'},
2263            can be used to correct for variations in naming or to merge several labels in one
2264        filter_annotated : bool, default False
2265            if True, the samples that do not have any labels will be filtered
2266        filter_background : bool, default False
2267            if True, only the unlabeled frames that are close to annotated frames will be labeled as background
2268        error_class : str, optional
2269            the name of the error class (the annotations that intersect with this label will be discarded)
2270        min_frames_action : int, default 0
2271            the minimum length of an action (shorter actions are not annotated)
2272        key_objects : tuple, optional
2273            the key objects to load the AnnotationStore from
2274        visibility_min_score : float, default 5
2275            the minimum visibility score for visibility filtering
2276        visibility_min_frac : float, default 0.7
2277            the minimum fraction of visible frames for visibility filtering
2278        mask : dict, optional
2279            a masked value dictionary (for active learning simulation experiments)
2280        use_hard_negatives : bool, default False
2281            mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing
2282        annotation_suffix : str | set, optional
2283            the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix}
2284            (not passed if creating from key objects or if irrelevant for the dataset)
2285        """
2286
2287        super().__init__(
2288            video_order=video_order,
2289            min_frames=min_frames,
2290            max_frames=max_frames,
2291            visibility=visibility,
2292            exclusive=exclusive,
2293            len_segment=len_segment,
2294            overlap=overlap,
2295            behaviors=behaviors,
2296            ignored_classes=ignored_classes,
2297            ignored_clips=ignored_clips,
2298            annotation_suffix=annotation_suffix,
2299            annotation_path=annotation_path,
2300            behavior_file=None,
2301            correction=correction,
2302            frame_limit=0,
2303            filter_annotated=filter_annotated,
2304            filter_background=filter_background,
2305            error_class=error_class,
2306            min_frames_action=min_frames_action,
2307            key_objects=key_objects,
2308            visibility_min_score=visibility_min_score,
2309            visibility_min_frac=visibility_min_frac,
2310            mask=mask,
2311            use_hard_negatives=use_hard_negatives,
2312            interactive=True,
2313        )

Parameters

video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) min_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip start frames (not passed if creating from key objects) max_frames : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are clip end frames (not passed if creating from key objects) visibility : dict, optional a nested dictionary where first-level keys are video ids, second-level keys are clip ids and values are visibility score arrays (not passed if creating from key objects or if irrelevant for the dataset) exclusive : bool, default True if True, the annotation is single-label; if False, multi-label len_segment : int, default 128 the length of the segments in which the data should be cut (in frames) overlap : int, default 0 the length of the overlap between neighboring segments (in frames) behaviors : set, optional the list of behaviors to put in the annotation (not passed if creating a blank instance or if behaviors are loaded from a file) ignored_classes : set, optional the list of behaviors from the behaviors list or file to not annotate ignored_clips : set, optional clip ids to ignore annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects) behavior_file : str, optional the path to an .xlsx behavior file (not passed if creating from key objects or if irrelevant for the dataset) correction : dict, optional a dictionary of corrections for the labels (e.g. {'sleping': 'sleeping', 'calm locomotion': 'locomotion'}, can be used to correct for variations in naming or to merge several labels in one filter_annotated : bool, default False if True, the samples that do not have any labels will be filtered filter_background : bool, default False if True, only the unlabeled frames that are close to annotated frames will be labeled as background error_class : str, optional the name of the error class (the annotations that intersect with this label will be discarded) min_frames_action : int, default 0 the minimum length of an action (shorter actions are not annotated) key_objects : tuple, optional the key objects to load the AnnotationStore from visibility_min_score : float, default 5 the minimum visibility score for visibility filtering visibility_min_frac : float, default 0.7 the minimum fraction of visible frames for visibility filtering mask : dict, optional a masked value dictionary (for active learning simulation experiments) use_hard_negatives : bool, default False mark hard negatives as 2 instead of 0 or 1, for loss functions that have options for hard negative processing annotation_suffix : str | set, optional the suffix or the set of suffices such that the annotation files are named {video_id}{annotation_suffix} (not passed if creating from key objects or if irrelevant for the dataset)