import argparseimport osimport torchfrom torchvision.datasets.utils import download_urlfrom diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, Transformer2DModelpretrained_models = {512: "DiT-XL-2-512x512.pt", 256: "DiT-XL-2-256x256.pt"}def download_model(model_name):"""Downloads a pre-trained DiT model from the web."""local_path = f"pretrained_models/{model_name}"if not os.path.isfile(local_path):os.makedirs("pretrained_models", exist_ok=True)web_path = f"https://dl.fbaipublicfiles.com/DiT/models/{model_name}"download_url(web_path, "pretrained_models")model = torch.load(local_path, map_location=lambda storage, loc: storage)return modeldef main(args):state_dict = download_model(pretrained_models[args.image_size])state_dict["pos_embed.proj.weight"] = state_dict["x_embedder.proj.weight"]state_dict["pos_embed.proj.bias"] = state_dict["x_embedder.proj.bias"]state_dict.pop("x_embedder.proj.weight")state_dict.pop("x_embedder.proj.bias")for depth in range(28):state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_1.weight"] = state_dict["t_embedder.mlp.0.weight"]state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_1.bias"] = state_dict["t_embedder.mlp.0.bias"]state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_2.weight"] = state_dict["t_embedder.mlp.2.weight"]state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_2.bias"] = state_dict["t_embedder.mlp.2.bias"]state_dict[f"transformer_blocks.{depth}.norm1.emb.class_embedder.embedding_table.weight"] = state_dict["y_embedder.embedding_table.weight"]state_dict[f"transformer_blocks.{depth}.norm1.linear.weight"] = state_dict[f"blocks.{depth}.adaLN_modulation.1.weight"]state_dict[f"transformer_blocks.{depth}.norm1.linear.bias"] = state_dict[f"blocks.{depth}.adaLN_modulation.1.bias"]q, k, v = torch.chunk(state_dict[f"blocks.{depth}.attn.qkv.weight"], 3, dim=0)q_bias, k_bias, v_bias = torch.chunk(state_dict[f"blocks.{depth}.attn.qkv.bias"], 3, dim=0)state_dict[f"transformer_blocks.{depth}.attn1.to_q.weight"] = qstate_dict[f"transformer_blocks.{depth}.attn1.to_q.bias"] = q_biasstate_dict[f"transformer_blocks.{depth}.attn1.to_k.weight"] = kstate_dict[f"transformer_blocks.{depth}.attn1.to_k.bias"] = k_biasstate_dict[f"transformer_blocks.{depth}.attn1.to_v.weight"] = vstate_dict[f"transformer_blocks.{depth}.attn1.to_v.bias"] = v_biasstate_dict[f"transformer_blocks.{depth}.attn1.to_out.0.weight"] = state_dict[f"blocks.{depth}.attn.proj.weight"]state_dict[f"transformer_blocks.{depth}.attn1.to_out.0.bias"] = state_dict[f"blocks.{depth}.attn.proj.bias"]state_dict[f"transformer_blocks.{depth}.ff.net.0.proj.weight"] = state_dict[f"blocks.{depth}.mlp.fc1.weight"]state_dict[f"transformer_blocks.{depth}.ff.net.0.proj.bias"] = state_dict[f"blocks.{depth}.mlp.fc1.bias"]state_dict[f"transformer_blocks.{depth}.ff.net.2.weight"] = state_dict[f"blocks.{depth}.mlp.fc2.weight"]state_dict[f"transformer_blocks.{depth}.ff.net.2.bias"] = state_dict[f"blocks.{depth}.mlp.fc2.bias"]state_dict.pop(f"blocks.{depth}.attn.qkv.weight")state_dict.pop(f"blocks.{depth}.attn.qkv.bias")state_dict.pop(f"blocks.{depth}.attn.proj.weight")state_dict.pop(f"blocks.{depth}.attn.proj.bias")state_dict.pop(f"blocks.{depth}.mlp.fc1.weight")state_dict.pop(f"blocks.{depth}.mlp.fc1.bias")state_dict.pop(f"blocks.{depth}.mlp.fc2.weight")state_dict.pop(f"blocks.{depth}.mlp.fc2.bias")state_dict.pop(f"blocks.{depth}.adaLN_modulation.1.weight")state_dict.pop(f"blocks.{depth}.adaLN_modulation.1.bias")state_dict.pop("t_embedder.mlp.0.weight")state_dict.pop("t_embedder.mlp.0.bias")state_dict.pop("t_embedder.mlp.2.weight")state_dict.pop("t_embedder.mlp.2.bias")state_dict.pop("y_embedder.embedding_table.weight")state_dict["proj_out_1.weight"] = state_dict["final_layer.adaLN_modulation.1.weight"]state_dict["proj_out_1.bias"] = state_dict["final_layer.adaLN_modulation.1.bias"]state_dict["proj_out_2.weight"] = state_dict["final_layer.linear.weight"]state_dict["proj_out_2.bias"] = state_dict["final_layer.linear.bias"]state_dict.pop("final_layer.linear.weight")state_dict.pop("final_layer.linear.bias")state_dict.pop("final_layer.adaLN_modulation.1.weight")state_dict.pop("final_layer.adaLN_modulation.1.bias")# DiT XL/2transformer = Transformer2DModel(sample_size=args.image_size // 8,num_layers=28,attention_head_dim=72,in_channels=4,out_channels=8,patch_size=2,attention_bias=True,num_attention_heads=16,activation_fn="gelu-approximate",num_embeds_ada_norm=1000,norm_type="ada_norm_zero",norm_elementwise_affine=False,)transformer.load_state_dict(state_dict, strict=True)scheduler = DDIMScheduler(num_train_timesteps=1000,beta_schedule="linear",prediction_type="epsilon",clip_sample=False,)vae = AutoencoderKL.from_pretrained(args.vae_model)pipeline = DiTPipeline(transformer=transformer, vae=vae, scheduler=scheduler)if args.save:pipeline.save_pretrained(args.checkpoint_path)if __name__ == "__main__":parser = argparse.ArgumentParser()parser.add_argument("--image_size",default=256,type=int,required=False,help="Image size of pretrained model, either 256 or 512.",)parser.add_argument("--vae_model",default="stabilityai/sd-vae-ft-ema",type=str,required=False,help="Path to pretrained VAE model, either stabilityai/sd-vae-ft-mse or stabilityai/sd-vae-ft-ema.",)parser.add_argument("--save", default=True, type=bool, required=False, help="Whether to save the converted pipeline or not.")parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output pipeline.")args = parser.parse_args()main(args)
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。