| import numpy as np
|
| from enum import Enum
|
|
|
| class BodyIdentifier(Enum):
|
| INDEX_FINGER_DIP_right = 0
|
| INDEX_FINGER_MCP_right = 1
|
| INDEX_FINGER_PIP_right = 2
|
| INDEX_FINGER_TIP_right = 3
|
| MIDDLE_FINGER_DIP_right = 4
|
| MIDDLE_FINGER_MCP_right = 5
|
| MIDDLE_FINGER_PIP_right = 6
|
| MIDDLE_FINGER_TIP_right = 7
|
| PINKY_DIP_right = 8
|
| PINKY_MCP_right = 9
|
| PINKY_PIP_right = 10
|
| PINKY_TIP_right = 11
|
| RING_FINGER_DIP_right = 12
|
| RING_FINGER_MCP_right = 13
|
| RING_FINGER_PIP_right = 14
|
| RING_FINGER_TIP_right = 15
|
| THUMB_CMC_right = 16
|
| THUMB_IP_right = 17
|
| THUMB_MCP_right = 18
|
| THUMB_TIP_right = 19
|
| WRIST_right = 20
|
| INDEX_FINGER_DIP_left = 21
|
| INDEX_FINGER_MCP_left = 22
|
| INDEX_FINGER_PIP_left = 23
|
| INDEX_FINGER_TIP_left = 24
|
| MIDDLE_FINGER_DIP_left = 25
|
| MIDDLE_FINGER_MCP_left = 26
|
| MIDDLE_FINGER_PIP_left = 27
|
| MIDDLE_FINGER_TIP_left = 28
|
| PINKY_DIP_left = 29
|
| PINKY_MCP_left = 30
|
| PINKY_PIP_left = 31
|
| PINKY_TIP_left = 32
|
| RING_FINGER_DIP_left = 33
|
| RING_FINGER_MCP_left = 34
|
| RING_FINGER_PIP_left = 35
|
| RING_FINGER_TIP_left = 36
|
| THUMB_CMC_left = 37
|
| THUMB_IP_left = 38
|
| THUMB_MCP_left = 39
|
| THUMB_TIP_left = 40
|
| WRIST_left = 41
|
| RIGHT_SHOULDER = 42
|
| LEFT_SHOULDER = 43
|
| LEFT_ELBOW = 44
|
| RIGHT_ELBOW = 45
|
|
|
| class Graph():
|
| """ The Graph to model the skeletons extracted by the openpose
|
|
|
| Args:
|
| strategy (string): must be one of the follow candidates
|
| - uniform: Uniform Labeling
|
| - distance: Distance Partitioning
|
| - spatial: Spatial Configuration
|
| For more information, please refer to the section 'Partition Strategies'
|
| in our paper (https://arxiv.org/abs/1801.07455).
|
|
|
| layout (string): must be one of the follow candidates
|
| - openpose: Is consists of 18 joints. For more information, please
|
| refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose#output
|
| - ntu-rgb+d: Is consists of 25 joints. For more information, please
|
| refer to https://github.com/shahroudy/NTURGB-D
|
|
|
| max_hop (int): the maximal distance between two connected nodes
|
| dilation (int): controls the spacing between the kernel points
|
|
|
| """
|
|
|
| def __init__(self,
|
| layout='openpose',
|
| strategy='uniform',
|
| max_hop=1,
|
| dilation=1):
|
| self.max_hop = max_hop
|
| self.dilation = dilation
|
|
|
| self.get_edge(layout)
|
| self.hop_dis = get_hop_distance(
|
| self.num_node, self.edge, max_hop=max_hop)
|
| self.get_adjacency(strategy)
|
|
|
| def __str__(self):
|
| return self.A
|
|
|
| def get_edge(self, layout):
|
| if layout == 'openpose':
|
| self.num_node = 18
|
| self_link = [(i, i) for i in range(self.num_node)]
|
| neighbor_link = [(4, 3), (3, 2), (7, 6), (6, 5), (13, 12), (12,
|
| 11),
|
| (10, 9), (9, 8), (11, 5), (8, 2), (5, 1), (2, 1),
|
| (0, 1), (15, 0), (14, 0), (17, 15), (16, 14)]
|
| self.edge = self_link + neighbor_link
|
| self.center = 1
|
| elif layout == 'ntu-rgb+d':
|
| self.num_node = 25
|
| self_link = [(i, i) for i in range(self.num_node)]
|
| neighbor_1base = [(1, 2), (2, 21), (3, 21), (4, 3), (5, 21),
|
| (6, 5), (7, 6), (8, 7), (9, 21), (10, 9),
|
| (11, 10), (12, 11), (13, 1), (14, 13), (15, 14),
|
| (16, 15), (17, 1), (18, 17), (19, 18), (20, 19),
|
| (22, 23), (23, 8), (24, 25), (25, 12)]
|
| neighbor_link = [(i - 1, j - 1) for (i, j) in neighbor_1base]
|
| self.edge = self_link + neighbor_link
|
| self.center = 21 - 1
|
| elif layout == 'ntu_edge':
|
| self.num_node = 24
|
| self_link = [(i, i) for i in range(self.num_node)]
|
| neighbor_1base = [(1, 2), (3, 2), (4, 3), (5, 2), (6, 5), (7, 6),
|
| (8, 7), (9, 2), (10, 9), (11, 10), (12, 11),
|
| (13, 1), (14, 13), (15, 14), (16, 15), (17, 1),
|
| (18, 17), (19, 18), (20, 19), (21, 22), (22, 8),
|
| (23, 24), (24, 12)]
|
| neighbor_link = [(i - 1, j - 1) for (i, j) in neighbor_1base]
|
| self.edge = self_link + neighbor_link
|
| self.center = 2
|
| elif layout == 'mediapipe':
|
| self.num_node = 25
|
| self_link = [(i, i) for i in range(self.num_node)]
|
| neighbor_1base = [(20, 18), (18, 16), (20, 16), (16, 22), (16, 14), (14, 12),
|
| (19, 17), (17, 15), (19, 15), (15, 21), (15, 13), (13, 11),
|
| (12, 11), (12, 24), (24, 23), (23, 11),
|
| (10, 9),
|
| (0, 4), (4, 5), (5, 6), (6, 8),
|
| (0, 1), (1, 2), (2, 3), (3, 7)]
|
| neighbor_link = [(i - 1, j - 1) for (i, j) in neighbor_1base]
|
| self.edge = self_link + neighbor_link
|
| self.center = 10
|
|
|
| elif layout == "mediapipe_two_hand":
|
| self.num_node = 46
|
| self_link = [(i, i) for i in range(self.num_node)]
|
| neighbor_1base = [(BodyIdentifier.WRIST_left.value, BodyIdentifier.THUMB_CMC_left.value),
|
| (BodyIdentifier.THUMB_CMC_left.value, BodyIdentifier.THUMB_MCP_left.value),
|
| (BodyIdentifier.THUMB_MCP_left.value, BodyIdentifier.THUMB_IP_left.value),
|
| (BodyIdentifier.THUMB_IP_left.value, BodyIdentifier.THUMB_TIP_left.value),
|
|
|
| (BodyIdentifier.WRIST_left.value, BodyIdentifier.INDEX_FINGER_MCP_left.value),
|
| (BodyIdentifier.INDEX_FINGER_MCP_left.value, BodyIdentifier.INDEX_FINGER_PIP_left.value),
|
| (BodyIdentifier.INDEX_FINGER_PIP_left.value, BodyIdentifier.INDEX_FINGER_DIP_left.value),
|
| (BodyIdentifier.INDEX_FINGER_DIP_left.value, BodyIdentifier.INDEX_FINGER_TIP_left.value),
|
|
|
| (BodyIdentifier.INDEX_FINGER_MCP_left.value, BodyIdentifier.MIDDLE_FINGER_MCP_left.value),
|
| (BodyIdentifier.MIDDLE_FINGER_MCP_left.value, BodyIdentifier.MIDDLE_FINGER_PIP_left.value),
|
| (BodyIdentifier.MIDDLE_FINGER_PIP_left.value, BodyIdentifier.MIDDLE_FINGER_DIP_left.value),
|
| (BodyIdentifier.MIDDLE_FINGER_DIP_left.value, BodyIdentifier.MIDDLE_FINGER_TIP_left.value),
|
|
|
| (BodyIdentifier.MIDDLE_FINGER_MCP_left.value, BodyIdentifier.RING_FINGER_MCP_left.value),
|
| (BodyIdentifier.RING_FINGER_MCP_left.value, BodyIdentifier.RING_FINGER_PIP_left.value),
|
| (BodyIdentifier.RING_FINGER_PIP_left.value, BodyIdentifier.RING_FINGER_DIP_left.value),
|
| (BodyIdentifier.RING_FINGER_DIP_left.value, BodyIdentifier.RING_FINGER_TIP_left.value),
|
|
|
| (BodyIdentifier.WRIST_left.value, BodyIdentifier.PINKY_MCP_left.value),
|
| (BodyIdentifier.PINKY_MCP_left.value, BodyIdentifier.PINKY_PIP_left.value),
|
| (BodyIdentifier.PINKY_PIP_left.value, BodyIdentifier.PINKY_DIP_left.value),
|
| (BodyIdentifier.PINKY_DIP_left.value, BodyIdentifier.PINKY_TIP_left.value),
|
|
|
|
|
| (BodyIdentifier.WRIST_right.value, BodyIdentifier.THUMB_CMC_right.value),
|
| (BodyIdentifier.THUMB_CMC_right.value, BodyIdentifier.THUMB_MCP_right.value),
|
| (BodyIdentifier.THUMB_MCP_right.value, BodyIdentifier.THUMB_IP_right.value),
|
| (BodyIdentifier.THUMB_IP_right.value, BodyIdentifier.THUMB_TIP_right.value),
|
|
|
| (BodyIdentifier.WRIST_right.value, BodyIdentifier.INDEX_FINGER_MCP_right.value),
|
| (BodyIdentifier.INDEX_FINGER_MCP_right.value, BodyIdentifier.INDEX_FINGER_PIP_right.value),
|
| (BodyIdentifier.INDEX_FINGER_PIP_right.value, BodyIdentifier.INDEX_FINGER_DIP_right.value),
|
| (BodyIdentifier.INDEX_FINGER_DIP_right.value, BodyIdentifier.INDEX_FINGER_TIP_right.value),
|
|
|
| (BodyIdentifier.INDEX_FINGER_MCP_right.value, BodyIdentifier.MIDDLE_FINGER_MCP_right.value),
|
| (BodyIdentifier.MIDDLE_FINGER_MCP_right.value, BodyIdentifier.MIDDLE_FINGER_PIP_right.value),
|
| (BodyIdentifier.MIDDLE_FINGER_PIP_right.value, BodyIdentifier.MIDDLE_FINGER_DIP_right.value),
|
| (BodyIdentifier.MIDDLE_FINGER_DIP_right.value, BodyIdentifier.MIDDLE_FINGER_TIP_right.value),
|
|
|
| (BodyIdentifier.MIDDLE_FINGER_MCP_right.value, BodyIdentifier.RING_FINGER_MCP_right.value),
|
| (BodyIdentifier.RING_FINGER_MCP_right.value, BodyIdentifier.RING_FINGER_PIP_right.value),
|
| (BodyIdentifier.RING_FINGER_PIP_right.value, BodyIdentifier.RING_FINGER_DIP_right.value),
|
| (BodyIdentifier.RING_FINGER_DIP_right.value, BodyIdentifier.RING_FINGER_TIP_right.value),
|
|
|
| (BodyIdentifier.WRIST_right.value, BodyIdentifier.PINKY_MCP_right.value),
|
| (BodyIdentifier.PINKY_MCP_right.value, BodyIdentifier.PINKY_PIP_right.value),
|
| (BodyIdentifier.PINKY_PIP_right.value, BodyIdentifier.PINKY_DIP_right.value),
|
| (BodyIdentifier.PINKY_DIP_right.value, BodyIdentifier.PINKY_TIP_right.value),
|
|
|
|
|
| (BodyIdentifier.RIGHT_SHOULDER.value, BodyIdentifier.RIGHT_ELBOW.value),
|
| (BodyIdentifier.RIGHT_ELBOW.value, BodyIdentifier.WRIST_right.value),
|
|
|
| (BodyIdentifier.RIGHT_SHOULDER.value, BodyIdentifier.LEFT_SHOULDER.value),
|
|
|
| (BodyIdentifier.LEFT_SHOULDER.value, BodyIdentifier.LEFT_ELBOW.value),
|
| (BodyIdentifier.LEFT_ELBOW.value, BodyIdentifier.WRIST_left.value)]
|
|
|
| neighbor_link = [(i, j) for (i, j) in neighbor_1base]
|
| self.edge = self_link + neighbor_link
|
| self.center = BodyIdentifier.RIGHT_SHOULDER.value
|
|
|
|
|
| else:
|
| raise ValueError("Do Not Exist This Layout.")
|
|
|
| def get_adjacency(self, strategy):
|
| valid_hop = range(0, self.max_hop + 1, self.dilation)
|
| adjacency = np.zeros((self.num_node, self.num_node))
|
| for hop in valid_hop:
|
| adjacency[self.hop_dis == hop] = 1
|
| normalize_adjacency = normalize_digraph(adjacency)
|
|
|
| if strategy == 'uniform':
|
| A = np.zeros((1, self.num_node, self.num_node))
|
| A[0] = normalize_adjacency
|
| self.A = A
|
| elif strategy == 'distance':
|
| A = np.zeros((len(valid_hop), self.num_node, self.num_node))
|
| for i, hop in enumerate(valid_hop):
|
| A[i][self.hop_dis == hop] = normalize_adjacency[self.hop_dis ==
|
| hop]
|
| self.A = A
|
| elif strategy == 'spatial':
|
| A = []
|
| for hop in valid_hop:
|
| a_root = np.zeros((self.num_node, self.num_node))
|
| a_close = np.zeros((self.num_node, self.num_node))
|
| a_further = np.zeros((self.num_node, self.num_node))
|
| for i in range(self.num_node):
|
| for j in range(self.num_node):
|
| if self.hop_dis[j, i] == hop:
|
| if self.hop_dis[j, self.center] == self.hop_dis[
|
| i, self.center]:
|
| a_root[j, i] = normalize_adjacency[j, i]
|
| elif self.hop_dis[j, self.
|
| center] > self.hop_dis[i, self.
|
| center]:
|
| a_close[j, i] = normalize_adjacency[j, i]
|
| else:
|
| a_further[j, i] = normalize_adjacency[j, i]
|
| if hop == 0:
|
| A.append(a_root)
|
| else:
|
| A.append(a_root + a_close)
|
| A.append(a_further)
|
| A = np.stack(A)
|
| self.A = A
|
| else:
|
| raise ValueError("Do Not Exist This Strategy")
|
|
|
|
|
| def get_hop_distance(num_node, edge, max_hop=1):
|
| A = np.zeros((num_node, num_node))
|
| print(edge)
|
| for i, j in edge:
|
| A[j, i] = 1
|
| A[i, j] = 1
|
|
|
|
|
| hop_dis = np.zeros((num_node, num_node)) + np.inf
|
| transfer_mat = [np.linalg.matrix_power(A, d) for d in range(max_hop + 1)]
|
| arrive_mat = (np.stack(transfer_mat) > 0)
|
| for d in range(max_hop, -1, -1):
|
| hop_dis[arrive_mat[d]] = d
|
| return hop_dis
|
|
|
|
|
| def normalize_digraph(A):
|
| Dl = np.sum(A, 0)
|
| num_node = A.shape[0]
|
| Dn = np.zeros((num_node, num_node))
|
| for i in range(num_node):
|
| if Dl[i] > 0:
|
| Dn[i, i] = Dl[i]**(-1)
|
| AD = np.dot(A, Dn)
|
| return AD
|
|
|
|
|
| def normalize_undigraph(A):
|
| Dl = np.sum(A, 0)
|
| num_node = A.shape[0]
|
| Dn = np.zeros((num_node, num_node))
|
| for i in range(num_node):
|
| if Dl[i] > 0:
|
| Dn[i, i] = Dl[i]**(-0.5)
|
| DAD = np.dot(np.dot(Dn, A), Dn)
|
| return DAD |
