dlc2action.data.base_store
Abstract parent classes for the store objects
1# 2# Copyright 2020-2022 by A. Mathis Group and contributors. All rights reserved. 3# 4# This project and all its files are licensed under GNU AGPLv3 or later version. A copy is included in dlc2action/LICENSE.AGPL. 5# 6""" 7Abstract parent classes for the store objects 8""" 9 10import os.path 11from typing import Dict, Union, List, Tuple, Set, Optional 12from abc import ABC, abstractmethod 13import numpy as np 14import inspect 15import torch 16 17 18class Store(ABC): # + 19 """ 20 A general parent class for `AnnotationStore` and `InputStore` 21 22 Processes input video information and generates ordered arrays of data samples and corresponding unique 23 original coordinates, as well as some meta objects. 24 It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). 25 Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). 26 The original coordinates object contains information about the video_id, clip_id and start time of the 27 samples in the original input data. 28 A Store has to be fully defined with a tuple of key objects. 29 The data array can be accessed with integer indices. 30 The samples can be stored as a tensor or TensorDict in RAM or as an array of file paths to be loaded on runtime. 31 """ 32 33 @abstractmethod 34 def __len__(self) -> int: 35 """ 36 Get the number of available samples 37 38 Returns 39 ------- 40 length : int 41 the number of available samples 42 """ 43 44 @abstractmethod 45 def remove(self, indices: List) -> None: 46 """ 47 Remove the samples corresponding to indices 48 49 Parameters 50 ---------- 51 indices : int 52 a list of integer indices to remove 53 """ 54 55 @abstractmethod 56 def key_objects(self) -> Tuple: 57 """ 58 Return a tuple of the key objects necessary to re-create the Store 59 60 Returns 61 ------- 62 key_objects : tuple 63 a tuple of key objects 64 """ 65 66 @abstractmethod 67 def load_from_key_objects(self, key_objects: Tuple) -> None: 68 """ 69 Load the information from a tuple of key objects 70 71 Parameters 72 ---------- 73 key_objects : tuple 74 a tuple of key objects 75 """ 76 77 @abstractmethod 78 def to_ram(self) -> None: 79 """ 80 Transfer the data samples to RAM if they were previously stored as file paths 81 """ 82 83 @abstractmethod 84 def get_original_coordinates(self) -> np.ndarray: 85 """ 86 Return the original coordinates array 87 88 Returns 89 ------- 90 np.ndarray 91 an array that contains the coordinates of the data samples in original input data (video id, clip id, 92 start frame) 93 """ 94 95 @abstractmethod 96 def create_subsample(self, indices: List, ssl_indices: List = None): 97 """ 98 Create a new store that contains a subsample of the data 99 100 Parameters 101 ---------- 102 indices : list 103 the indices to be included in the subsample 104 ssl_indices : list, optional 105 the indices to be included in the subsample without the annotation data 106 """ 107 108 @classmethod 109 @abstractmethod 110 def get_file_ids(cls, *args, **kwargs) -> List: 111 """ 112 Process data parameters and return a list of ids of the videos that should 113 be processed by the __init__ function 114 115 Returns 116 ------- 117 video_ids : list 118 a list of video file ids 119 """ 120 121 @classmethod 122 def get_parameters(cls) -> List: 123 """ 124 Generate a list of parameter names for the __init__ function 125 126 Returns 127 ------- 128 parameter_names: list 129 a list of necessary parameter names 130 """ 131 132 return inspect.getfullargspec(cls.__init__).args 133 134 @classmethod 135 def new(cls): 136 """ 137 Create a new instance of the same class 138 139 Returns 140 ------- 141 new_instance : Store 142 a new instance of the same class 143 """ 144 145 return cls() 146 147 148class InputStore(Store): # + 149 """ 150 A class that generates model input data from video information and stores it 151 152 Processes input video information and generates ordered arrays of data samples and corresponding unique 153 original coordinates, as well as some meta objects. 154 It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). 155 Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). 156 The original coordinates object contains information about the video_id, clip_id and start time of the 157 samples in the original input data. 158 An InputStore has to be fully defined with a tuple of key objects. 159 The data array can be accessed with integer indices. 160 The samples can be stored as a TensorDict in RAM or as an array of file paths to be loaded on runtime. 161 When no arguments are passed a blank class instance should be created that can later be filled with 162 information from key objects 163 """ 164 165 @abstractmethod 166 def __init__( 167 self, 168 video_order: List = None, 169 key_objects: Tuple = None, 170 data_path: Union[str, List] = None, 171 file_paths: List = None, 172 feature_save_path: str = None, 173 feature_extraction_pars: Dict = None, 174 *args, 175 **kwargs 176 ): 177 """ 178 Parameters 179 ---------- 180 video_order : list, optional 181 a list of video ids that should be processed in the same order (not passed if creating from key objects) 182 key_objects : tuple, optional 183 a tuple of key objects 184 data_path : str | set, optional 185 the path to the folder where the pose and feature files are stored or a set of such paths 186 (not passed if creating from key objects or from `file_paths`) 187 file_paths : set, optional 188 a set of string paths to the pose and feature files 189 (not passed if creating from key objects or from `data_path`) 190 feature_save_path : str, optional 191 the path to the folder where pre-processed files are stored (not passed if creating from key objects) 192 feature_extraction_pars : dict, optional 193 a dictionary of feature extraction parameters (not passed if creating from key objects) 194 """ 195 196 if key_objects is not None: 197 self.load_from_key_objects(key_objects) 198 199 @abstractmethod 200 def __getitem__(self, ind: int) -> Dict: 201 """ 202 Return the sample corresponding to an index 203 204 Parameters 205 ---------- 206 ind : int 207 index of the sample 208 209 Returns 210 ------- 211 sample : dict 212 the corresponding sample (a dictionary of features) 213 """ 214 215 @abstractmethod 216 def get_video_id(self, coords: Tuple) -> str: 217 """ 218 Get the video id from an element of original coordinates 219 220 Parameters 221 ---------- 222 coords : tuple 223 an element of the original coordinates array 224 225 Returns 226 ------- 227 video_id: str 228 the id of the video that the coordinates point to 229 """ 230 231 @abstractmethod 232 def get_clip_id(self, coords: Tuple) -> str: 233 """ 234 Get the clip id from an element of original coordinates 235 236 Parameters 237 ---------- 238 coords : tuple 239 an element of the original coordinates array 240 241 Returns 242 ------- 243 clip_id : str 244 the id of the clip that the coordinates point to 245 """ 246 247 @abstractmethod 248 def get_clip_length(self, video_id: str, clip_id: str) -> int: 249 """ 250 Get the clip length from the id 251 252 Parameters 253 ---------- 254 video_id : str 255 the video id 256 clip_id : str 257 the clip id 258 259 Returns 260 ------- 261 clip_length : int 262 the length of the clip 263 """ 264 265 @abstractmethod 266 def get_clip_start_end(self, coords: Tuple) -> Tuple[int, int]: 267 """ 268 Get the clip start and end frames from an element of original coordinates 269 270 Parameters 271 ---------- 272 coords : tuple 273 an element of original coordinates array 274 275 Returns 276 ------- 277 start: int 278 the start frame of the clip that the coordinates point to 279 end : int 280 the end frame of the clip that the coordinates point to 281 """ 282 283 @abstractmethod 284 def get_clip_start(self, video_id: str, clip_id: str) -> int: 285 """ 286 Get the clip start frame from the video id and the clip id 287 288 Parameters 289 ---------- 290 video_id : str 291 the video id 292 clip_id : str 293 the clip id 294 295 Returns 296 ------- 297 clip_start : int 298 the start frame of the clip 299 """ 300 301 @abstractmethod 302 def get_visibility( 303 self, video_id: str, clip_id: str, start: int, end: int, score: float 304 ) -> float: 305 """ 306 Get the fraction of the frames in that have a visibility score better than a hard_threshold 307 308 For example, in the case of keypoint data the visibility score can be the number of identified keypoints. 309 310 Parameters 311 ---------- 312 video_id : str 313 the video id of the frames 314 clip_id : str 315 the clip id of the frames 316 start : int 317 the start frame 318 end : int 319 the end frame 320 score : float 321 the visibility score hard_threshold 322 323 Returns 324 ------- 325 frac_visible: float 326 the fraction of frames with visibility above the hard_threshold 327 """ 328 329 @abstractmethod 330 def get_annotation_objects(self) -> Dict: 331 """ 332 Get a dictionary of objects necessary to create an AnnotationStore 333 334 Returns 335 ------- 336 annotation_objects : dict 337 a dictionary of objects to be passed to the AnnotationStore constructor where the keys are the names of 338 the objects 339 """ 340 341 @abstractmethod 342 def get_folder(self, video_id: str) -> str: 343 """ 344 Get the input folder that the file with this video id was read from 345 346 Parameters 347 ---------- 348 video_id : str 349 the video id 350 351 Returns 352 ------- 353 folder : str 354 the path to the directory that contains the input file associated with the video id 355 """ 356 357 def get_clip_length_from_coords(self, coords: Tuple) -> int: 358 """ 359 Get the length of a clip from an element of the original coordinates array 360 361 Parameters 362 ---------- 363 coords : tuple 364 an element of the original coordinates array 365 366 Returns 367 ------- 368 clip_length : int 369 the length of the clip 370 """ 371 372 v_id = self.get_video_id(coords) 373 clip_id = self.get_clip_id(coords) 374 l = self.get_clip_length(v_id, clip_id) 375 return l 376 377 def get_folder_order(self) -> List: 378 """ 379 Get a list of folders corresponding to the data samples 380 381 Returns 382 ------- 383 folder_order : list 384 a list of string folder basenames corresponding to the data samples (e.g. 'folder2' 385 if the corresponding file was read from '/path/to/folder1/folder2') 386 """ 387 388 return [os.path.basename(self.get_folder(x)) for x in self.get_video_id_order()] 389 390 def get_video_id_order(self) -> List: 391 """ 392 Get a list of video ids corresponding to the data samples 393 394 Returns 395 ------- 396 video_id_order : list 397 a list of string names of the video ids corresponding to the data samples 398 """ 399 400 return [self.get_video_id(x) for x in self.get_original_coordinates()] 401 402 def get_tag(self, idx: int) -> Union[int, None]: 403 """ 404 Return a tag object corresponding to an index 405 406 Tags can carry meta information (like annotator id) and are accepted by models that require 407 that information and by metrics (some metrics have options for averaging over the tags). 408 When a tag is `None`, it is not passed to the model. 409 410 Parameters 411 ---------- 412 idx : int 413 the index 414 415 Returns 416 ------- 417 tag : int 418 the tag index 419 """ 420 421 return None 422 423 def get_indices(self, tag: int) -> List: 424 """ 425 Get a list of indices of samples that have a specific meta tag 426 427 Parameters 428 ---------- 429 tag : int 430 the meta tag for the subsample (`None` for the whole dataset) 431 432 Returns 433 ------- 434 indices : list 435 a list of indices that meet the criteria 436 """ 437 438 return list(range(len(self))) 439 440 def get_tags(self) -> List: 441 """ 442 Get a list of all meta tags 443 444 Returns 445 ------- 446 tags: List 447 a list of unique meta tag values 448 """ 449 450 return [None] 451 452 453class AnnotationStore(Store): 454 """ 455 A class that generates annotation from video information and stores it 456 457 Processes input video information and generates ordered arrays of annotation samples and corresponding unique 458 original coordinates, as well as some meta objects. 459 It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). 460 Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). 461 The original coordinates object contains information about the video_id, clip_id and start time of the 462 samples in the original input data. 463 An AnnotationStore has to be fully defined with a tuple of key objects. 464 The annotation array can be accessed with integer indices. 465 The samples can be stored as a torch.Tensor in RAM or as an array of file paths to be loaded on runtime. 466 When no arguments are passed a blank class instance should be created that can later be filled with 467 information from key objects 468 """ 469 470 required_objects = [] 471 """ 472 A list of string names of the objects required from the input store 473 """ 474 475 @abstractmethod 476 def __init__( 477 self, 478 video_order: List = None, 479 key_objects: Tuple = None, 480 annotation_path: Union[str, Set] = None, 481 *args, 482 **kwargs 483 ): 484 """ 485 Parameters 486 ---------- 487 video_order : list, optional 488 a list of video ids that should be processed in the same order (not passed if creating from key objects) 489 key_objects : tuple, optional 490 a tuple of key objects 491 annotation_path : str | set, optional 492 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 493 from key objects) 494 """ 495 496 if key_objects is not None: 497 self.load_from_key_objects(key_objects) 498 499 @abstractmethod 500 def __getitem__(self, ind: int) -> torch.Tensor: 501 """ 502 Return the annotation of the sample corresponding to an index 503 504 Parameters 505 ---------- 506 ind : int 507 index of the sample 508 509 Returns 510 ------- 511 sample : torch.Tensor 512 the corresponding annotation tensor 513 """ 514 515 @abstractmethod 516 def get_len(self, return_unlabeled: bool) -> int: 517 """ 518 Get the length of the subsample of labeled/unlabeled data 519 520 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 521 and if return_unlabeled is None the index is already correct 522 523 Parameters 524 ---------- 525 return_unlabeled : bool 526 the identifier for the subsample 527 528 Returns 529 ------- 530 length : int 531 the length of the subsample 532 """ 533 534 @abstractmethod 535 def count_classes( 536 self, frac: bool = False, zeros: bool = False, bouts: bool = False 537 ) -> Dict: 538 """ 539 Get a dictionary with class-wise frame counts 540 541 Parameters 542 ---------- 543 frac : bool, default False 544 if `True`, a fraction of the total frame count is returned 545 zeros : bool. default False 546 if `True`, the number of known negative samples is counted (only if the annotation is multi-label) 547 bouts : bool, default False 548 if `True`, instead of frame counts segment counts are returned 549 550 Returns 551 ------- 552 count_dictionary : dict 553 a dictionary with class indices as keys and frame counts as values 554 555 """ 556 557 @abstractmethod 558 def behaviors_dict(self) -> Dict: 559 """ 560 Get a dictionary of class names 561 562 Returns 563 ------- 564 behavior_dictionary: dict 565 a dictionary with class indices as keys and class names as values 566 """ 567 568 @abstractmethod 569 def annotation_class(self) -> str: 570 """ 571 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon) 572 573 Returns 574 ------- 575 annotation_class : str 576 the type of annotation 577 """ 578 579 @abstractmethod 580 def size(self) -> int: 581 """ 582 Get the total number of frames in the data 583 584 Returns 585 ------- 586 size : int 587 the total number of frames 588 """ 589 590 @abstractmethod 591 def filtered_indices(self) -> List: 592 """ 593 Return the indices of the samples that should be removed 594 595 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 596 parameters 597 598 Returns 599 ------- 600 indices_to_remove : list 601 a list of integer indices that should be removed 602 """ 603 604 @abstractmethod 605 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 606 """ 607 Set pseudo labels to the unlabeled data 608 609 Parameters 610 ---------- 611 labels : torch.Tensor 612 a tensor of pseudo-labels for the unlabeled data 613 """ 614 615 616class PoseInputStore(InputStore): 617 """ 618 A subclass of InputStore for pose estimation data 619 620 Contains methods used by pose estimation feature extractors. 621 All methods receive a data dictionary as input. This dictionary is the same as what is passed to the 622 feature extractor and the only limitations for the structure are that it has to relate to one video id 623 and have clip ids as keys. Read the documentation at `dlc2action.data` to find out more about videos 624 and clips. 625 """ 626 627 def get_likelihood( 628 self, data_dict: Dict, clip_id: str, bodypart: str 629 ) -> Union[np.ndarray, None]: 630 """ 631 Get the likelihood values 632 633 Parameters 634 ---------- 635 data_dict : dict 636 the data dictionary 637 clip_id : str 638 the clip id 639 bodypart : str 640 the name of the body part 641 642 Returns 643 ------- 644 likelihoods: np.ndarrray | None 645 `None` if the dataset doesn't have likelihoods or an array of shape (#timestamps) 646 """ 647 648 return None 649 650 @abstractmethod 651 def get_coords(self, data_dict: Dict, clip_id: str, bodypart: str) -> np.ndarray: 652 """ 653 Get the coordinates array of a specific body part in a specific clip 654 655 Parameters 656 ---------- 657 data_dict : dict 658 the data dictionary 659 clip_id : str 660 the clip id 661 bodypart : str 662 the name of the body part 663 664 Returns 665 ------- 666 coords : np.ndarray 667 the coordinates array of shape (#timesteps, #coordinates) 668 """ 669 670 @abstractmethod 671 def get_bodyparts(self) -> List: 672 """ 673 Get a list of bodypart names 674 675 Returns 676 ------- 677 bodyparts : list 678 a list of string or integer body part names 679 """ 680 681 @abstractmethod 682 def get_n_frames(self, data_dict: Dict, clip_id: str) -> int: 683 """ 684 Get the length of the clip 685 686 Parameters 687 ---------- 688 data_dict : dict 689 the data dictionary 690 clip_id : str 691 the clip id 692 693 Returns 694 ------- 695 n_frames : int 696 the length of the clip 697 """
class
Store(abc.ABC):
19class Store(ABC): # + 20 """ 21 A general parent class for `AnnotationStore` and `InputStore` 22 23 Processes input video information and generates ordered arrays of data samples and corresponding unique 24 original coordinates, as well as some meta objects. 25 It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). 26 Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). 27 The original coordinates object contains information about the video_id, clip_id and start time of the 28 samples in the original input data. 29 A Store has to be fully defined with a tuple of key objects. 30 The data array can be accessed with integer indices. 31 The samples can be stored as a tensor or TensorDict in RAM or as an array of file paths to be loaded on runtime. 32 """ 33 34 @abstractmethod 35 def __len__(self) -> int: 36 """ 37 Get the number of available samples 38 39 Returns 40 ------- 41 length : int 42 the number of available samples 43 """ 44 45 @abstractmethod 46 def remove(self, indices: List) -> None: 47 """ 48 Remove the samples corresponding to indices 49 50 Parameters 51 ---------- 52 indices : int 53 a list of integer indices to remove 54 """ 55 56 @abstractmethod 57 def key_objects(self) -> Tuple: 58 """ 59 Return a tuple of the key objects necessary to re-create the Store 60 61 Returns 62 ------- 63 key_objects : tuple 64 a tuple of key objects 65 """ 66 67 @abstractmethod 68 def load_from_key_objects(self, key_objects: Tuple) -> None: 69 """ 70 Load the information from a tuple of key objects 71 72 Parameters 73 ---------- 74 key_objects : tuple 75 a tuple of key objects 76 """ 77 78 @abstractmethod 79 def to_ram(self) -> None: 80 """ 81 Transfer the data samples to RAM if they were previously stored as file paths 82 """ 83 84 @abstractmethod 85 def get_original_coordinates(self) -> np.ndarray: 86 """ 87 Return the original coordinates array 88 89 Returns 90 ------- 91 np.ndarray 92 an array that contains the coordinates of the data samples in original input data (video id, clip id, 93 start frame) 94 """ 95 96 @abstractmethod 97 def create_subsample(self, indices: List, ssl_indices: List = None): 98 """ 99 Create a new store that contains a subsample of the data 100 101 Parameters 102 ---------- 103 indices : list 104 the indices to be included in the subsample 105 ssl_indices : list, optional 106 the indices to be included in the subsample without the annotation data 107 """ 108 109 @classmethod 110 @abstractmethod 111 def get_file_ids(cls, *args, **kwargs) -> List: 112 """ 113 Process data parameters and return a list of ids of the videos that should 114 be processed by the __init__ function 115 116 Returns 117 ------- 118 video_ids : list 119 a list of video file ids 120 """ 121 122 @classmethod 123 def get_parameters(cls) -> List: 124 """ 125 Generate a list of parameter names for the __init__ function 126 127 Returns 128 ------- 129 parameter_names: list 130 a list of necessary parameter names 131 """ 132 133 return inspect.getfullargspec(cls.__init__).args 134 135 @classmethod 136 def new(cls): 137 """ 138 Create a new instance of the same class 139 140 Returns 141 ------- 142 new_instance : Store 143 a new instance of the same class 144 """ 145 146 return cls()
A general parent class for AnnotationStore and InputStore
Processes input video information and generates ordered arrays of data samples and corresponding unique original coordinates, as well as some meta objects. It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). The original coordinates object contains information about the video_id, clip_id and start time of the samples in the original input data. A Store has to be fully defined with a tuple of key objects. The data array can be accessed with integer indices. The samples can be stored as a tensor or TensorDict in RAM or as an array of file paths to be loaded on runtime.
@abstractmethod
def
remove(self, indices: List) -> None:
45 @abstractmethod 46 def remove(self, indices: List) -> None: 47 """ 48 Remove the samples corresponding to indices 49 50 Parameters 51 ---------- 52 indices : int 53 a list of integer indices to remove 54 """
Remove the samples corresponding to indices
Parameters
indices : int a list of integer indices to remove
@abstractmethod
def
key_objects(self) -> Tuple:
56 @abstractmethod 57 def key_objects(self) -> Tuple: 58 """ 59 Return a tuple of the key objects necessary to re-create the Store 60 61 Returns 62 ------- 63 key_objects : tuple 64 a tuple of key objects 65 """
Return a tuple of the key objects necessary to re-create the Store
Returns
key_objects : tuple a tuple of key objects
@abstractmethod
def
load_from_key_objects(self, key_objects: Tuple) -> None:
67 @abstractmethod 68 def load_from_key_objects(self, key_objects: Tuple) -> None: 69 """ 70 Load the information from a tuple of key objects 71 72 Parameters 73 ---------- 74 key_objects : tuple 75 a tuple of key objects 76 """
Load the information from a tuple of key objects
Parameters
key_objects : tuple a tuple of key objects
@abstractmethod
def
to_ram(self) -> None:
78 @abstractmethod 79 def to_ram(self) -> None: 80 """ 81 Transfer the data samples to RAM if they were previously stored as file paths 82 """
Transfer the data samples to RAM if they were previously stored as file paths
@abstractmethod
def
get_original_coordinates(self) -> numpy.ndarray:
84 @abstractmethod 85 def get_original_coordinates(self) -> np.ndarray: 86 """ 87 Return the original coordinates array 88 89 Returns 90 ------- 91 np.ndarray 92 an array that contains the coordinates of the data samples in original input data (video id, clip id, 93 start frame) 94 """
Return the original coordinates array
Returns
np.ndarray an array that contains the coordinates of the data samples in original input data (video id, clip id, start frame)
@abstractmethod
def
create_subsample(self, indices: List, ssl_indices: List = None)
96 @abstractmethod 97 def create_subsample(self, indices: List, ssl_indices: List = None): 98 """ 99 Create a new store that contains a subsample of the data 100 101 Parameters 102 ---------- 103 indices : list 104 the indices to be included in the subsample 105 ssl_indices : list, optional 106 the indices to be included in the subsample without the annotation data 107 """
Create a new store that contains a subsample of the data
Parameters
indices : list the indices to be included in the subsample ssl_indices : list, optional the indices to be included in the subsample without the annotation data
@classmethod
@abstractmethod
def
get_file_ids(cls, *args, **kwargs) -> List:
109 @classmethod 110 @abstractmethod 111 def get_file_ids(cls, *args, **kwargs) -> List: 112 """ 113 Process data parameters and return a list of ids of the videos that should 114 be processed by the __init__ function 115 116 Returns 117 ------- 118 video_ids : list 119 a list of video file ids 120 """
Process data parameters and return a list of ids of the videos that should be processed by the __init__ function
Returns
video_ids : list a list of video file ids
@classmethod
def
get_parameters(cls) -> List:
122 @classmethod 123 def get_parameters(cls) -> List: 124 """ 125 Generate a list of parameter names for the __init__ function 126 127 Returns 128 ------- 129 parameter_names: list 130 a list of necessary parameter names 131 """ 132 133 return inspect.getfullargspec(cls.__init__).args
Generate a list of parameter names for the __init__ function
Returns
parameter_names: list a list of necessary parameter names
@classmethod
def
new(cls)
135 @classmethod 136 def new(cls): 137 """ 138 Create a new instance of the same class 139 140 Returns 141 ------- 142 new_instance : Store 143 a new instance of the same class 144 """ 145 146 return cls()
Create a new instance of the same class
Returns
new_instance : Store a new instance of the same class
149class InputStore(Store): # + 150 """ 151 A class that generates model input data from video information and stores it 152 153 Processes input video information and generates ordered arrays of data samples and corresponding unique 154 original coordinates, as well as some meta objects. 155 It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). 156 Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). 157 The original coordinates object contains information about the video_id, clip_id and start time of the 158 samples in the original input data. 159 An InputStore has to be fully defined with a tuple of key objects. 160 The data array can be accessed with integer indices. 161 The samples can be stored as a TensorDict in RAM or as an array of file paths to be loaded on runtime. 162 When no arguments are passed a blank class instance should be created that can later be filled with 163 information from key objects 164 """ 165 166 @abstractmethod 167 def __init__( 168 self, 169 video_order: List = None, 170 key_objects: Tuple = None, 171 data_path: Union[str, List] = None, 172 file_paths: List = None, 173 feature_save_path: str = None, 174 feature_extraction_pars: Dict = None, 175 *args, 176 **kwargs 177 ): 178 """ 179 Parameters 180 ---------- 181 video_order : list, optional 182 a list of video ids that should be processed in the same order (not passed if creating from key objects) 183 key_objects : tuple, optional 184 a tuple of key objects 185 data_path : str | set, optional 186 the path to the folder where the pose and feature files are stored or a set of such paths 187 (not passed if creating from key objects or from `file_paths`) 188 file_paths : set, optional 189 a set of string paths to the pose and feature files 190 (not passed if creating from key objects or from `data_path`) 191 feature_save_path : str, optional 192 the path to the folder where pre-processed files are stored (not passed if creating from key objects) 193 feature_extraction_pars : dict, optional 194 a dictionary of feature extraction parameters (not passed if creating from key objects) 195 """ 196 197 if key_objects is not None: 198 self.load_from_key_objects(key_objects) 199 200 @abstractmethod 201 def __getitem__(self, ind: int) -> Dict: 202 """ 203 Return the sample corresponding to an index 204 205 Parameters 206 ---------- 207 ind : int 208 index of the sample 209 210 Returns 211 ------- 212 sample : dict 213 the corresponding sample (a dictionary of features) 214 """ 215 216 @abstractmethod 217 def get_video_id(self, coords: Tuple) -> str: 218 """ 219 Get the video id from an element of original coordinates 220 221 Parameters 222 ---------- 223 coords : tuple 224 an element of the original coordinates array 225 226 Returns 227 ------- 228 video_id: str 229 the id of the video that the coordinates point to 230 """ 231 232 @abstractmethod 233 def get_clip_id(self, coords: Tuple) -> str: 234 """ 235 Get the clip id from an element of original coordinates 236 237 Parameters 238 ---------- 239 coords : tuple 240 an element of the original coordinates array 241 242 Returns 243 ------- 244 clip_id : str 245 the id of the clip that the coordinates point to 246 """ 247 248 @abstractmethod 249 def get_clip_length(self, video_id: str, clip_id: str) -> int: 250 """ 251 Get the clip length from the id 252 253 Parameters 254 ---------- 255 video_id : str 256 the video id 257 clip_id : str 258 the clip id 259 260 Returns 261 ------- 262 clip_length : int 263 the length of the clip 264 """ 265 266 @abstractmethod 267 def get_clip_start_end(self, coords: Tuple) -> Tuple[int, int]: 268 """ 269 Get the clip start and end frames from an element of original coordinates 270 271 Parameters 272 ---------- 273 coords : tuple 274 an element of original coordinates array 275 276 Returns 277 ------- 278 start: int 279 the start frame of the clip that the coordinates point to 280 end : int 281 the end frame of the clip that the coordinates point to 282 """ 283 284 @abstractmethod 285 def get_clip_start(self, video_id: str, clip_id: str) -> int: 286 """ 287 Get the clip start frame from the video id and the clip id 288 289 Parameters 290 ---------- 291 video_id : str 292 the video id 293 clip_id : str 294 the clip id 295 296 Returns 297 ------- 298 clip_start : int 299 the start frame of the clip 300 """ 301 302 @abstractmethod 303 def get_visibility( 304 self, video_id: str, clip_id: str, start: int, end: int, score: float 305 ) -> float: 306 """ 307 Get the fraction of the frames in that have a visibility score better than a hard_threshold 308 309 For example, in the case of keypoint data the visibility score can be the number of identified keypoints. 310 311 Parameters 312 ---------- 313 video_id : str 314 the video id of the frames 315 clip_id : str 316 the clip id of the frames 317 start : int 318 the start frame 319 end : int 320 the end frame 321 score : float 322 the visibility score hard_threshold 323 324 Returns 325 ------- 326 frac_visible: float 327 the fraction of frames with visibility above the hard_threshold 328 """ 329 330 @abstractmethod 331 def get_annotation_objects(self) -> Dict: 332 """ 333 Get a dictionary of objects necessary to create an AnnotationStore 334 335 Returns 336 ------- 337 annotation_objects : dict 338 a dictionary of objects to be passed to the AnnotationStore constructor where the keys are the names of 339 the objects 340 """ 341 342 @abstractmethod 343 def get_folder(self, video_id: str) -> str: 344 """ 345 Get the input folder that the file with this video id was read from 346 347 Parameters 348 ---------- 349 video_id : str 350 the video id 351 352 Returns 353 ------- 354 folder : str 355 the path to the directory that contains the input file associated with the video id 356 """ 357 358 def get_clip_length_from_coords(self, coords: Tuple) -> int: 359 """ 360 Get the length of a clip from an element of the original coordinates array 361 362 Parameters 363 ---------- 364 coords : tuple 365 an element of the original coordinates array 366 367 Returns 368 ------- 369 clip_length : int 370 the length of the clip 371 """ 372 373 v_id = self.get_video_id(coords) 374 clip_id = self.get_clip_id(coords) 375 l = self.get_clip_length(v_id, clip_id) 376 return l 377 378 def get_folder_order(self) -> List: 379 """ 380 Get a list of folders corresponding to the data samples 381 382 Returns 383 ------- 384 folder_order : list 385 a list of string folder basenames corresponding to the data samples (e.g. 'folder2' 386 if the corresponding file was read from '/path/to/folder1/folder2') 387 """ 388 389 return [os.path.basename(self.get_folder(x)) for x in self.get_video_id_order()] 390 391 def get_video_id_order(self) -> List: 392 """ 393 Get a list of video ids corresponding to the data samples 394 395 Returns 396 ------- 397 video_id_order : list 398 a list of string names of the video ids corresponding to the data samples 399 """ 400 401 return [self.get_video_id(x) for x in self.get_original_coordinates()] 402 403 def get_tag(self, idx: int) -> Union[int, None]: 404 """ 405 Return a tag object corresponding to an index 406 407 Tags can carry meta information (like annotator id) and are accepted by models that require 408 that information and by metrics (some metrics have options for averaging over the tags). 409 When a tag is `None`, it is not passed to the model. 410 411 Parameters 412 ---------- 413 idx : int 414 the index 415 416 Returns 417 ------- 418 tag : int 419 the tag index 420 """ 421 422 return None 423 424 def get_indices(self, tag: int) -> List: 425 """ 426 Get a list of indices of samples that have a specific meta tag 427 428 Parameters 429 ---------- 430 tag : int 431 the meta tag for the subsample (`None` for the whole dataset) 432 433 Returns 434 ------- 435 indices : list 436 a list of indices that meet the criteria 437 """ 438 439 return list(range(len(self))) 440 441 def get_tags(self) -> List: 442 """ 443 Get a list of all meta tags 444 445 Returns 446 ------- 447 tags: List 448 a list of unique meta tag values 449 """ 450 451 return [None]
A class that generates model input data from video information and stores it
Processes input video information and generates ordered arrays of data samples and corresponding unique original coordinates, as well as some meta objects. It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). The original coordinates object contains information about the video_id, clip_id and start time of the samples in the original input data. An InputStore has to be fully defined with a tuple of key objects. The data array can be accessed with integer indices. The samples can be stored as a TensorDict in RAM or as an array of file paths to be loaded on runtime. When no arguments are passed a blank class instance should be created that can later be filled with information from key objects
@abstractmethod
InputStore( video_order: List = None, key_objects: Tuple = None, data_path: Union[str, List] = None, file_paths: List = None, feature_save_path: str = None, feature_extraction_pars: Dict = None, *args, **kwargs)
166 @abstractmethod 167 def __init__( 168 self, 169 video_order: List = None, 170 key_objects: Tuple = None, 171 data_path: Union[str, List] = None, 172 file_paths: List = None, 173 feature_save_path: str = None, 174 feature_extraction_pars: Dict = None, 175 *args, 176 **kwargs 177 ): 178 """ 179 Parameters 180 ---------- 181 video_order : list, optional 182 a list of video ids that should be processed in the same order (not passed if creating from key objects) 183 key_objects : tuple, optional 184 a tuple of key objects 185 data_path : str | set, optional 186 the path to the folder where the pose and feature files are stored or a set of such paths 187 (not passed if creating from key objects or from `file_paths`) 188 file_paths : set, optional 189 a set of string paths to the pose and feature files 190 (not passed if creating from key objects or from `data_path`) 191 feature_save_path : str, optional 192 the path to the folder where pre-processed files are stored (not passed if creating from key objects) 193 feature_extraction_pars : dict, optional 194 a dictionary of feature extraction parameters (not passed if creating from key objects) 195 """ 196 197 if key_objects is not None: 198 self.load_from_key_objects(key_objects)
Parameters
video_order : list, optional
a list of video ids that should be processed in the same order (not passed if creating from key objects)
key_objects : tuple, optional
a tuple of key objects
data_path : str | set, optional
the path to the folder where the pose and feature files are stored or a set of such paths
(not passed if creating from key objects or from file_paths)
file_paths : set, optional
a set of string paths to the pose and feature files
(not passed if creating from key objects or from data_path)
feature_save_path : str, optional
the path to the folder where pre-processed files are stored (not passed if creating from key objects)
feature_extraction_pars : dict, optional
a dictionary of feature extraction parameters (not passed if creating from key objects)
@abstractmethod
def
get_video_id(self, coords: Tuple) -> str:
216 @abstractmethod 217 def get_video_id(self, coords: Tuple) -> str: 218 """ 219 Get the video id from an element of original coordinates 220 221 Parameters 222 ---------- 223 coords : tuple 224 an element of the original coordinates array 225 226 Returns 227 ------- 228 video_id: str 229 the id of the video that the coordinates point to 230 """
Get the video id from an element of original coordinates
Parameters
coords : tuple an element of the original coordinates array
Returns
video_id: str the id of the video that the coordinates point to
@abstractmethod
def
get_clip_id(self, coords: Tuple) -> str:
232 @abstractmethod 233 def get_clip_id(self, coords: Tuple) -> str: 234 """ 235 Get the clip id from an element of original coordinates 236 237 Parameters 238 ---------- 239 coords : tuple 240 an element of the original coordinates array 241 242 Returns 243 ------- 244 clip_id : str 245 the id of the clip that the coordinates point to 246 """
Get the clip id from an element of original coordinates
Parameters
coords : tuple an element of the original coordinates array
Returns
clip_id : str the id of the clip that the coordinates point to
@abstractmethod
def
get_clip_length(self, video_id: str, clip_id: str) -> int:
248 @abstractmethod 249 def get_clip_length(self, video_id: str, clip_id: str) -> int: 250 """ 251 Get the clip length from the id 252 253 Parameters 254 ---------- 255 video_id : str 256 the video id 257 clip_id : str 258 the clip id 259 260 Returns 261 ------- 262 clip_length : int 263 the length of the clip 264 """
Get the clip length from the id
Parameters
video_id : str the video id clip_id : str the clip id
Returns
clip_length : int the length of the clip
@abstractmethod
def
get_clip_start_end(self, coords: Tuple) -> Tuple[int, int]:
266 @abstractmethod 267 def get_clip_start_end(self, coords: Tuple) -> Tuple[int, int]: 268 """ 269 Get the clip start and end frames from an element of original coordinates 270 271 Parameters 272 ---------- 273 coords : tuple 274 an element of original coordinates array 275 276 Returns 277 ------- 278 start: int 279 the start frame of the clip that the coordinates point to 280 end : int 281 the end frame of the clip that the coordinates point to 282 """
Get the clip start and end frames from an element of original coordinates
Parameters
coords : tuple an element of original coordinates array
Returns
start: int the start frame of the clip that the coordinates point to end : int the end frame of the clip that the coordinates point to
@abstractmethod
def
get_clip_start(self, video_id: str, clip_id: str) -> int:
284 @abstractmethod 285 def get_clip_start(self, video_id: str, clip_id: str) -> int: 286 """ 287 Get the clip start frame from the video id and the clip id 288 289 Parameters 290 ---------- 291 video_id : str 292 the video id 293 clip_id : str 294 the clip id 295 296 Returns 297 ------- 298 clip_start : int 299 the start frame of the clip 300 """
Get the clip start frame from the video id and the clip id
Parameters
video_id : str the video id clip_id : str the clip id
Returns
clip_start : int the start frame of the clip
@abstractmethod
def
get_visibility( self, video_id: str, clip_id: str, start: int, end: int, score: float) -> float:
302 @abstractmethod 303 def get_visibility( 304 self, video_id: str, clip_id: str, start: int, end: int, score: float 305 ) -> float: 306 """ 307 Get the fraction of the frames in that have a visibility score better than a hard_threshold 308 309 For example, in the case of keypoint data the visibility score can be the number of identified keypoints. 310 311 Parameters 312 ---------- 313 video_id : str 314 the video id of the frames 315 clip_id : str 316 the clip id of the frames 317 start : int 318 the start frame 319 end : int 320 the end frame 321 score : float 322 the visibility score hard_threshold 323 324 Returns 325 ------- 326 frac_visible: float 327 the fraction of frames with visibility above the hard_threshold 328 """
Get the fraction of the frames in that have a visibility score better than a hard_threshold
For example, in the case of keypoint data the visibility score can be the number of identified keypoints.
Parameters
video_id : str the video id of the frames clip_id : str the clip id of the frames start : int the start frame end : int the end frame score : float the visibility score hard_threshold
Returns
frac_visible: float the fraction of frames with visibility above the hard_threshold
@abstractmethod
def
get_annotation_objects(self) -> Dict:
330 @abstractmethod 331 def get_annotation_objects(self) -> Dict: 332 """ 333 Get a dictionary of objects necessary to create an AnnotationStore 334 335 Returns 336 ------- 337 annotation_objects : dict 338 a dictionary of objects to be passed to the AnnotationStore constructor where the keys are the names of 339 the objects 340 """
Get a dictionary of objects necessary to create an AnnotationStore
Returns
annotation_objects : dict a dictionary of objects to be passed to the AnnotationStore constructor where the keys are the names of the objects
@abstractmethod
def
get_folder(self, video_id: str) -> str:
342 @abstractmethod 343 def get_folder(self, video_id: str) -> str: 344 """ 345 Get the input folder that the file with this video id was read from 346 347 Parameters 348 ---------- 349 video_id : str 350 the video id 351 352 Returns 353 ------- 354 folder : str 355 the path to the directory that contains the input file associated with the video id 356 """
Get the input folder that the file with this video id was read from
Parameters
video_id : str the video id
Returns
folder : str the path to the directory that contains the input file associated with the video id
def
get_clip_length_from_coords(self, coords: Tuple) -> int:
358 def get_clip_length_from_coords(self, coords: Tuple) -> int: 359 """ 360 Get the length of a clip from an element of the original coordinates array 361 362 Parameters 363 ---------- 364 coords : tuple 365 an element of the original coordinates array 366 367 Returns 368 ------- 369 clip_length : int 370 the length of the clip 371 """ 372 373 v_id = self.get_video_id(coords) 374 clip_id = self.get_clip_id(coords) 375 l = self.get_clip_length(v_id, clip_id) 376 return l
Get the length of a clip from an element of the original coordinates array
Parameters
coords : tuple an element of the original coordinates array
Returns
clip_length : int the length of the clip
def
get_folder_order(self) -> List:
378 def get_folder_order(self) -> List: 379 """ 380 Get a list of folders corresponding to the data samples 381 382 Returns 383 ------- 384 folder_order : list 385 a list of string folder basenames corresponding to the data samples (e.g. 'folder2' 386 if the corresponding file was read from '/path/to/folder1/folder2') 387 """ 388 389 return [os.path.basename(self.get_folder(x)) for x in self.get_video_id_order()]
Get a list of folders corresponding to the data samples
Returns
folder_order : list a list of string folder basenames corresponding to the data samples (e.g. 'folder2' if the corresponding file was read from '/path/to/folder1/folder2')
def
get_video_id_order(self) -> List:
391 def get_video_id_order(self) -> List: 392 """ 393 Get a list of video ids corresponding to the data samples 394 395 Returns 396 ------- 397 video_id_order : list 398 a list of string names of the video ids corresponding to the data samples 399 """ 400 401 return [self.get_video_id(x) for x in self.get_original_coordinates()]
Get a list of video ids corresponding to the data samples
Returns
video_id_order : list a list of string names of the video ids corresponding to the data samples
def
get_tag(self, idx: int) -> Optional[int]:
403 def get_tag(self, idx: int) -> Union[int, None]: 404 """ 405 Return a tag object corresponding to an index 406 407 Tags can carry meta information (like annotator id) and are accepted by models that require 408 that information and by metrics (some metrics have options for averaging over the tags). 409 When a tag is `None`, it is not passed to the model. 410 411 Parameters 412 ---------- 413 idx : int 414 the index 415 416 Returns 417 ------- 418 tag : int 419 the tag index 420 """ 421 422 return None
Return a tag object corresponding to an index
Tags can carry meta information (like annotator id) and are accepted by models that require
that information and by metrics (some metrics have options for averaging over the tags).
When a tag is None, it is not passed to the model.
Parameters
idx : int the index
Returns
tag : int the tag index
def
get_indices(self, tag: int) -> List:
424 def get_indices(self, tag: int) -> List: 425 """ 426 Get a list of indices of samples that have a specific meta tag 427 428 Parameters 429 ---------- 430 tag : int 431 the meta tag for the subsample (`None` for the whole dataset) 432 433 Returns 434 ------- 435 indices : list 436 a list of indices that meet the criteria 437 """ 438 439 return list(range(len(self)))
Get a list of indices of samples that have a specific meta tag
Parameters
tag : int
the meta tag for the subsample (None for the whole dataset)
Returns
indices : list a list of indices that meet the criteria
454class AnnotationStore(Store): 455 """ 456 A class that generates annotation from video information and stores it 457 458 Processes input video information and generates ordered arrays of annotation samples and corresponding unique 459 original coordinates, as well as some meta objects. 460 It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). 461 Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). 462 The original coordinates object contains information about the video_id, clip_id and start time of the 463 samples in the original input data. 464 An AnnotationStore has to be fully defined with a tuple of key objects. 465 The annotation array can be accessed with integer indices. 466 The samples can be stored as a torch.Tensor in RAM or as an array of file paths to be loaded on runtime. 467 When no arguments are passed a blank class instance should be created that can later be filled with 468 information from key objects 469 """ 470 471 required_objects = [] 472 """ 473 A list of string names of the objects required from the input store 474 """ 475 476 @abstractmethod 477 def __init__( 478 self, 479 video_order: List = None, 480 key_objects: Tuple = None, 481 annotation_path: Union[str, Set] = None, 482 *args, 483 **kwargs 484 ): 485 """ 486 Parameters 487 ---------- 488 video_order : list, optional 489 a list of video ids that should be processed in the same order (not passed if creating from key objects) 490 key_objects : tuple, optional 491 a tuple of key objects 492 annotation_path : str | set, optional 493 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 494 from key objects) 495 """ 496 497 if key_objects is not None: 498 self.load_from_key_objects(key_objects) 499 500 @abstractmethod 501 def __getitem__(self, ind: int) -> torch.Tensor: 502 """ 503 Return the annotation of the sample corresponding to an index 504 505 Parameters 506 ---------- 507 ind : int 508 index of the sample 509 510 Returns 511 ------- 512 sample : torch.Tensor 513 the corresponding annotation tensor 514 """ 515 516 @abstractmethod 517 def get_len(self, return_unlabeled: bool) -> int: 518 """ 519 Get the length of the subsample of labeled/unlabeled data 520 521 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 522 and if return_unlabeled is None the index is already correct 523 524 Parameters 525 ---------- 526 return_unlabeled : bool 527 the identifier for the subsample 528 529 Returns 530 ------- 531 length : int 532 the length of the subsample 533 """ 534 535 @abstractmethod 536 def count_classes( 537 self, frac: bool = False, zeros: bool = False, bouts: bool = False 538 ) -> Dict: 539 """ 540 Get a dictionary with class-wise frame counts 541 542 Parameters 543 ---------- 544 frac : bool, default False 545 if `True`, a fraction of the total frame count is returned 546 zeros : bool. default False 547 if `True`, the number of known negative samples is counted (only if the annotation is multi-label) 548 bouts : bool, default False 549 if `True`, instead of frame counts segment counts are returned 550 551 Returns 552 ------- 553 count_dictionary : dict 554 a dictionary with class indices as keys and frame counts as values 555 556 """ 557 558 @abstractmethod 559 def behaviors_dict(self) -> Dict: 560 """ 561 Get a dictionary of class names 562 563 Returns 564 ------- 565 behavior_dictionary: dict 566 a dictionary with class indices as keys and class names as values 567 """ 568 569 @abstractmethod 570 def annotation_class(self) -> str: 571 """ 572 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon) 573 574 Returns 575 ------- 576 annotation_class : str 577 the type of annotation 578 """ 579 580 @abstractmethod 581 def size(self) -> int: 582 """ 583 Get the total number of frames in the data 584 585 Returns 586 ------- 587 size : int 588 the total number of frames 589 """ 590 591 @abstractmethod 592 def filtered_indices(self) -> List: 593 """ 594 Return the indices of the samples that should be removed 595 596 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 597 parameters 598 599 Returns 600 ------- 601 indices_to_remove : list 602 a list of integer indices that should be removed 603 """ 604 605 @abstractmethod 606 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 607 """ 608 Set pseudo labels to the unlabeled data 609 610 Parameters 611 ---------- 612 labels : torch.Tensor 613 a tensor of pseudo-labels for the unlabeled data 614 """
A class that generates annotation from video information and stores it
Processes input video information and generates ordered arrays of annotation samples and corresponding unique original coordinates, as well as some meta objects. It is assumed that the input videos are separated into clips (e.g. corresponding to different individuals). Each video and each clip inside the video has a unique id (video_id and clip_id, correspondingly). The original coordinates object contains information about the video_id, clip_id and start time of the samples in the original input data. An AnnotationStore has to be fully defined with a tuple of key objects. The annotation array can be accessed with integer indices. The samples can be stored as a torch.Tensor in RAM or as an array of file paths to be loaded on runtime. When no arguments are passed a blank class instance should be created that can later be filled with information from key objects
@abstractmethod
AnnotationStore( video_order: List = None, key_objects: Tuple = None, annotation_path: Union[str, Set] = None, *args, **kwargs)
476 @abstractmethod 477 def __init__( 478 self, 479 video_order: List = None, 480 key_objects: Tuple = None, 481 annotation_path: Union[str, Set] = None, 482 *args, 483 **kwargs 484 ): 485 """ 486 Parameters 487 ---------- 488 video_order : list, optional 489 a list of video ids that should be processed in the same order (not passed if creating from key objects) 490 key_objects : tuple, optional 491 a tuple of key objects 492 annotation_path : str | set, optional 493 the path or the set of paths to the folder where the annotation files are stored (not passed if creating 494 from key objects) 495 """ 496 497 if key_objects is not None: 498 self.load_from_key_objects(key_objects)
Parameters
video_order : list, optional a list of video ids that should be processed in the same order (not passed if creating from key objects) key_objects : tuple, optional a tuple of key objects annotation_path : str | set, optional the path or the set of paths to the folder where the annotation files are stored (not passed if creating from key objects)
@abstractmethod
def
get_len(self, return_unlabeled: bool) -> int:
516 @abstractmethod 517 def get_len(self, return_unlabeled: bool) -> int: 518 """ 519 Get the length of the subsample of labeled/unlabeled data 520 521 If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled 522 and if return_unlabeled is None the index is already correct 523 524 Parameters 525 ---------- 526 return_unlabeled : bool 527 the identifier for the subsample 528 529 Returns 530 ------- 531 length : int 532 the length of the subsample 533 """
Get the length of the subsample of labeled/unlabeled data
If return_unlabeled is True, the index is in the subsample of unlabeled data, if False in labeled and if return_unlabeled is None the index is already correct
Parameters
return_unlabeled : bool the identifier for the subsample
Returns
length : int the length of the subsample
@abstractmethod
def
count_classes( self, frac: bool = False, zeros: bool = False, bouts: bool = False) -> Dict:
535 @abstractmethod 536 def count_classes( 537 self, frac: bool = False, zeros: bool = False, bouts: bool = False 538 ) -> Dict: 539 """ 540 Get a dictionary with class-wise frame counts 541 542 Parameters 543 ---------- 544 frac : bool, default False 545 if `True`, a fraction of the total frame count is returned 546 zeros : bool. default False 547 if `True`, the number of known negative samples is counted (only if the annotation is multi-label) 548 bouts : bool, default False 549 if `True`, instead of frame counts segment counts are returned 550 551 Returns 552 ------- 553 count_dictionary : dict 554 a dictionary with class indices as keys and frame counts as values 555 556 """
Get a dictionary with class-wise frame counts
Parameters
frac : bool, default False
if True, a fraction of the total frame count is returned
zeros : bool. default False
if True, the number of known negative samples is counted (only if the annotation is multi-label)
bouts : bool, default False
if True, instead of frame counts segment counts are returned
Returns
count_dictionary : dict a dictionary with class indices as keys and frame counts as values
@abstractmethod
def
behaviors_dict(self) -> Dict:
558 @abstractmethod 559 def behaviors_dict(self) -> Dict: 560 """ 561 Get a dictionary of class names 562 563 Returns 564 ------- 565 behavior_dictionary: dict 566 a dictionary with class indices as keys and class names as values 567 """
Get a dictionary of class names
Returns
behavior_dictionary: dict a dictionary with class indices as keys and class names as values
@abstractmethod
def
annotation_class(self) -> str:
569 @abstractmethod 570 def annotation_class(self) -> str: 571 """ 572 Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon) 573 574 Returns 575 ------- 576 annotation_class : str 577 the type of annotation 578 """
Get the type of annotation ('exclusive_classification', 'nonexclusive_classification', more coming soon)
Returns
annotation_class : str the type of annotation
@abstractmethod
def
size(self) -> int:
580 @abstractmethod 581 def size(self) -> int: 582 """ 583 Get the total number of frames in the data 584 585 Returns 586 ------- 587 size : int 588 the total number of frames 589 """
Get the total number of frames in the data
Returns
size : int the total number of frames
@abstractmethod
def
filtered_indices(self) -> List:
591 @abstractmethod 592 def filtered_indices(self) -> List: 593 """ 594 Return the indices of the samples that should be removed 595 596 Choosing the indices can be based on any kind of filering defined in the __init__ function by the data 597 parameters 598 599 Returns 600 ------- 601 indices_to_remove : list 602 a list of integer indices that should be removed 603 """
Return the indices of the samples that should be removed
Choosing the indices can be based on any kind of filering defined in the __init__ function by the data parameters
Returns
indices_to_remove : list a list of integer indices that should be removed
@abstractmethod
def
set_pseudo_labels(self, labels: torch.Tensor) -> None:
605 @abstractmethod 606 def set_pseudo_labels(self, labels: torch.Tensor) -> None: 607 """ 608 Set pseudo labels to the unlabeled data 609 610 Parameters 611 ---------- 612 labels : torch.Tensor 613 a tensor of pseudo-labels for the unlabeled data 614 """
Set pseudo labels to the unlabeled data
Parameters
labels : torch.Tensor a tensor of pseudo-labels for the unlabeled data
617class PoseInputStore(InputStore): 618 """ 619 A subclass of InputStore for pose estimation data 620 621 Contains methods used by pose estimation feature extractors. 622 All methods receive a data dictionary as input. This dictionary is the same as what is passed to the 623 feature extractor and the only limitations for the structure are that it has to relate to one video id 624 and have clip ids as keys. Read the documentation at `dlc2action.data` to find out more about videos 625 and clips. 626 """ 627 628 def get_likelihood( 629 self, data_dict: Dict, clip_id: str, bodypart: str 630 ) -> Union[np.ndarray, None]: 631 """ 632 Get the likelihood values 633 634 Parameters 635 ---------- 636 data_dict : dict 637 the data dictionary 638 clip_id : str 639 the clip id 640 bodypart : str 641 the name of the body part 642 643 Returns 644 ------- 645 likelihoods: np.ndarrray | None 646 `None` if the dataset doesn't have likelihoods or an array of shape (#timestamps) 647 """ 648 649 return None 650 651 @abstractmethod 652 def get_coords(self, data_dict: Dict, clip_id: str, bodypart: str) -> np.ndarray: 653 """ 654 Get the coordinates array of a specific body part in a specific clip 655 656 Parameters 657 ---------- 658 data_dict : dict 659 the data dictionary 660 clip_id : str 661 the clip id 662 bodypart : str 663 the name of the body part 664 665 Returns 666 ------- 667 coords : np.ndarray 668 the coordinates array of shape (#timesteps, #coordinates) 669 """ 670 671 @abstractmethod 672 def get_bodyparts(self) -> List: 673 """ 674 Get a list of bodypart names 675 676 Returns 677 ------- 678 bodyparts : list 679 a list of string or integer body part names 680 """ 681 682 @abstractmethod 683 def get_n_frames(self, data_dict: Dict, clip_id: str) -> int: 684 """ 685 Get the length of the clip 686 687 Parameters 688 ---------- 689 data_dict : dict 690 the data dictionary 691 clip_id : str 692 the clip id 693 694 Returns 695 ------- 696 n_frames : int 697 the length of the clip 698 """
A subclass of InputStore for pose estimation data
Contains methods used by pose estimation feature extractors.
All methods receive a data dictionary as input. This dictionary is the same as what is passed to the
feature extractor and the only limitations for the structure are that it has to relate to one video id
and have clip ids as keys. Read the documentation at dlc2action.data to find out more about videos
and clips.
def
get_likelihood( self, data_dict: Dict, clip_id: str, bodypart: str) -> Optional[numpy.ndarray]:
628 def get_likelihood( 629 self, data_dict: Dict, clip_id: str, bodypart: str 630 ) -> Union[np.ndarray, None]: 631 """ 632 Get the likelihood values 633 634 Parameters 635 ---------- 636 data_dict : dict 637 the data dictionary 638 clip_id : str 639 the clip id 640 bodypart : str 641 the name of the body part 642 643 Returns 644 ------- 645 likelihoods: np.ndarrray | None 646 `None` if the dataset doesn't have likelihoods or an array of shape (#timestamps) 647 """ 648 649 return None
Get the likelihood values
Parameters
data_dict : dict the data dictionary clip_id : str the clip id bodypart : str the name of the body part
Returns
likelihoods: np.ndarrray | None
None if the dataset doesn't have likelihoods or an array of shape (#timestamps)
@abstractmethod
def
get_coords(self, data_dict: Dict, clip_id: str, bodypart: str) -> numpy.ndarray:
651 @abstractmethod 652 def get_coords(self, data_dict: Dict, clip_id: str, bodypart: str) -> np.ndarray: 653 """ 654 Get the coordinates array of a specific body part in a specific clip 655 656 Parameters 657 ---------- 658 data_dict : dict 659 the data dictionary 660 clip_id : str 661 the clip id 662 bodypart : str 663 the name of the body part 664 665 Returns 666 ------- 667 coords : np.ndarray 668 the coordinates array of shape (#timesteps, #coordinates) 669 """
Get the coordinates array of a specific body part in a specific clip
Parameters
data_dict : dict the data dictionary clip_id : str the clip id bodypart : str the name of the body part
Returns
coords : np.ndarray the coordinates array of shape (#timesteps, #coordinates)
@abstractmethod
def
get_bodyparts(self) -> List:
671 @abstractmethod 672 def get_bodyparts(self) -> List: 673 """ 674 Get a list of bodypart names 675 676 Returns 677 ------- 678 bodyparts : list 679 a list of string or integer body part names 680 """
Get a list of bodypart names
Returns
bodyparts : list a list of string or integer body part names
@abstractmethod
def
get_n_frames(self, data_dict: Dict, clip_id: str) -> int:
682 @abstractmethod 683 def get_n_frames(self, data_dict: Dict, clip_id: str) -> int: 684 """ 685 Get the length of the clip 686 687 Parameters 688 ---------- 689 data_dict : dict 690 the data dictionary 691 clip_id : str 692 the clip id 693 694 Returns 695 ------- 696 n_frames : int 697 the length of the clip 698 """
Get the length of the clip
Parameters
data_dict : dict the data dictionary clip_id : str the clip id
Returns
n_frames : int the length of the clip