20250506 meeting

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# Training configuration
batch_size: 1  # Batch size
output_directory: "./results/real_time"  # Output directory for checkpoints and logs
ckpt_iter: "max"  # Checkpoint mode (max or min)
iters_per_ckpt: 1000  # Checkpoint frequency (number of epochs)
iters_per_logging: 100  # Log frequency (number of iterations)
n_iters: 20000  # Maximum number of iterations
learning_rate: 0.001  # Learning rate

# Additional training settings
only_generate_missing: true  # Generate missing values only
use_model: 2  # Model to use for training
masking: "rm"  # Masking strategy for missing values
missing_k: 200  # Number of missing values

# Data paths
data:
  train_path: "./datasets/real_time/pollutants_train.npy"  # Path to training data

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

wavenet:
  # WaveNet model parameters
  input_channels: 26  # Number of input channels
  output_channels: 26  # Number of output channels
  residual_layers: 36  # Number of residual layers
  residual_channels: 256  # Number of channels in residual blocks
  skip_channels: 256  # Number of channels in skip connections

  # Diffusion step embedding dimensions
  diffusion_step_embed_dim_input: 128  # Input dimension
  diffusion_step_embed_dim_hidden: 512  # Middle dimension
  diffusion_step_embed_dim_output: 512  # Output dimension

  # Structured State Spaces sequence model (S4) configurations
  s4_max_sequence_length: 2000  # Maximum sequence length
  s4_state_dim: 64  # State dimension
  s4_dropout: 0.0  # Dropout rate
  s4_bidirectional: true  # Whether to use bidirectional layers
  s4_use_layer_norm: true  # Whether to use layer normalization

diffusion:
  # Diffusion model parameters
  T: 200  # Number of diffusion steps
  beta_0: 0.0001  # Initial beta value
  beta_T: 0.02  # Final beta value

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# Inference configuration
batch_size: 1  # Batch size for inference
output_directory: "./results/real_time/12/inference/mnr"  # Output directory for inference results
ckpt_path: "./results/real_time"  # Path to checkpoint for inference
trials: 1 # Replications

# Additional training settings
only_generate_missing: true  # Generate missing values only
use_model: 2  # Model to use for training
masking: "mnr"  # Masking strategy for missing values
missing_k: 12  # Number of missing values

# Data paths
data:
  test_path: "./datasets/real_time/pollutants_test.npy"  # Path to test data

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

def run_job(
    model_config: dict,
    inference_config: dict,
    device: Optional[Union[torch.device, str]],
    ckpt_iter: Union[str, int],
) -> None:

    ...
    dataloader = get_dataloader(
        inference_config["data"]["test_path"],
        batch_size,
        device=device,
    )
    ...

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

def get_dataloader(
    path: str,
    batch_size: int,
    is_shuffle: bool = True,
    device: Union[str, torch.device] = "cpu",
    num_workers: int = 0,
) -> DataLoader:
    """
    Get a PyTorch DataLoader for the dataset stored at the given path.

    Args:
        path (str): Path to the dataset file.
        batch_size (int): Size of each batch.
        is_shuffle (bool, optional): Whether to shuffle the dataset. Defaults to True.
        device (Union[str, torch.device], optional): Device to move the data to. Defaults to "cpu".
        num_workers (int, optional): Number of subprocesses to use for data loading. Defaults to 8.

    Returns:
        DataLoader: PyTorch DataLoader for the dataset.
    """
    dataset = TensorDataset(torch.from_numpy(np.load(path)).to(dtype=torch.float32))
    pin_memory = device == "cuda" or device == torch.device("cuda")
    return DataLoader(
        dataset,
        batch_size=batch_size,
        shuffle=is_shuffle,
        pin_memory=pin_memory,
        num_workers=num_workers,
    )

1
2
3
4

def generate(self) -> list:
    ...
    for index, (batch,) in enumerate(self.dataloader):
    ...

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

# test imputation
import os
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm

test_data = np.load(r'real_time\pollutants_test.npy').transpose(0, 2, 1)

## functions
def imputation_plot(full_data, missing_data, imputation_data, title, save_path):
    """
    Plot the first 3 dimensions of the first sample, comparing full data and missing data.
    """
    os.makedirs(save_path, exist_ok=True)
    show_dims = 2
    fig, axes = plt.subplots(show_dims, 1, figsize=(12, 8), sharex=True)
    #imputation_data = np.where(np.isnan(missing_data), imputation_data, np.nan)
    for j in range(show_dims):
        axes[j].plot(full_data[0, j], color='gray', label='full data', alpha=0.6)
        axes[j].plot(imputation_data[0, j], color='orange', label='imputation data', alpha=0.6)
        axes[j].plot(missing_data[0, j], color='red', label=title)
        axes[j].set_ylabel(f'Dim {j}')
        axes[j].legend()

    plt.suptitle(title)
    plt.xlabel('Time')
    plt.tight_layout()
    plt.savefig(f"{save_path}.png", dpi=300)
    #plt.show()

def imputation_plot_each_dim(full_data, missing_data, imputation_data, title, save_dir):
    """
    Plot each dimension of the first sample separately, comparing full data, missing data, and imputation data.
    Save each plot as a separate file.
    """
    num_dims = full_data.shape[1]
    os.makedirs(save_dir, exist_ok=True)  

    for j in tqdm(range(num_dims)):
        fig, ax = plt.subplots(figsize=(12, 4))
        ax.plot(full_data[0, j], color='gray', label='full data', alpha=0.6)
        ax.plot(imputation_data[0, j], color='orange', label='imputation data', alpha=0.6)
        ax.plot(missing_data[0, j], color='red', label=title)
        ax.set_ylabel(f'Dim {j}')
        ax.set_xlabel('Time')
        ax.set_title(f'{title} - Dimension {j}')

        ax.legend(
            loc='center left', 
            bbox_to_anchor=(1, 0.5) 
        )

        fig.tight_layout(rect=[0, 0, 0.85, 1])
        save_path = os.path.join(save_dir, f"{title}_dim{j}.png")
        plt.savefig(save_path, dpi=300)
        plt.close(fig)

def analysis_predict_data(missing_k, path):
    folder_path = f'.\\real_time\\imputation\\{missing_k}\\predict\\{path}\\T200_beta00.0001_betaT0.02\\max'
    os.makedirs(folder_path, exist_ok=True)
    length = len(os.listdir(folder_path))
    imputation = []
    for i in range(length):
        file_path = os.path.join(folder_path, f'imputation{i}.npy')
        arr = np.load(file_path)
        imputation.append(arr)
    predict_data = np.concatenate(imputation, axis=0)
    print(f'Shape: {predict_data.shape}')
    print(f'NAs: {np.isnan(predict_data).sum()}')
    # print(predict_data[0, 0])
    print(f'MSPE for all: {((test_data - predict_data)**2).mean()}')

    test_data_predict = np.load(f'real_time\\{missing_k}\\pollutants_test_{path}.npy').transpose(0, 2, 1)
    print(f'MSPE only for missing: {((test_data[np.isnan(test_data_predict)] - predict_data[np.isnan(test_data_predict)])**2).mean()}')

    imputation_plot(test_data, test_data_predict, predict_data, f'imputation {path} test data', f'real_time\\imputation\\{missing_k}\\result\\predict\\imputation0_{path}')
    imputation_plot_each_dim(test_data, test_data_predict, predict_data, f'imputation {path} test data', f'real_time\\imputation\\{missing_k}\\result\\predict\\imputation0_{path}')
    # test_data[0, 0][1:10]
    # test_data_predict[0, 0][1:10]
    # predict_data[0, 0][1:10]

missing_k = 200

## rm
analysis_predict_data(missing_k, 'rm')

## rbm
analysis_predict_data(missing_k, 'rbm')

## bm
analysis_predict_data(missing_k, 'bm')

## tf
analysis_predict_data(missing_k, 'tf')



# original
def analysis_imputation_data(missing_k, path):
    folder_path = f'.\\real_time\\imputation\\{missing_k}\\inference\\{path}\\T200_beta00.0001_betaT0.02\\max'
    os.makedirs(folder_path, exist_ok=True)
    length = len(os.listdir(folder_path))
    imputation = []
    for i in range(length):
        file_path = os.path.join(folder_path, f'imputation{i}.npy')
        arr = np.load(file_path)
        imputation.append(arr)
    imputation_data = np.concatenate(imputation, axis=0)
    print(f'Shape: {imputation_data.shape}')
    print(f'NAs: {np.isnan(imputation_data).sum()}')
    # print(imputation_data[0, 0])
    print(f'MSPE: {((test_data - imputation_data)**2).mean()}')

    imputation_plot(test_data, test_data, imputation_data, f'imputation {path} test data', f'real_time\\imputation\\{missing_k}\\result\\original\\imputation0_{path}')
    imputation_plot_each_dim(test_data, test_data, imputation_data, f'imputation {path} test data', f'real_time\\imputation\\{missing_k}\\result\\original\\imputation0_{path}')
    # test_data[0, 0][1:10]
    # test_data_imputation[0, 0][1:10]
    # imputation_data[0, 0][1:10]

## rm
analysis_imputation_data(missing_k, 'rm')

## bm
analysis_imputation_data(missing_k, 'bm')

## mnr
analysis_imputation_data(missing_k, 'mnr')

## tf
analysis_imputation_data(missing_k, 'tf')