mirror of
https://github.com/Fenrir784/e621tagger.git
synced 2026-11-20 00:31:18 -08:00
322 lines
9.1 KiB
Python
322 lines
9.1 KiB
Python
import re
|
|
from math import sqrt
|
|
from typing import Any, Iterable, MutableSequence, Self, cast
|
|
|
|
import torch
|
|
from torch import Tensor
|
|
from torch.nn import (
|
|
Module, ModuleList, Parameter, Buffer,
|
|
Linear, LayerNorm, RMSNorm, Dropout, Flatten,
|
|
init
|
|
)
|
|
from torch.nn.functional import scaled_dot_product_attention, normalize, linear
|
|
|
|
try:
|
|
from torch.nn.attention.varlen import varlen_attn
|
|
except ImportError:
|
|
pass
|
|
|
|
from einops import rearrange
|
|
|
|
from .glu import GLU, SwiGLU, SpGLU
|
|
|
|
__all__ = ("HydraPool",)
|
|
|
|
class _FeedForward(Module):
|
|
def __init__(
|
|
self,
|
|
input_dim: int,
|
|
hidden_dim: int,
|
|
output_dim: int = -1,
|
|
*,
|
|
glu: type[GLU],
|
|
affine: bool = False,
|
|
dropout: float = 0.0,
|
|
device: torch.device | str | None,
|
|
dtype: torch.dtype | None,
|
|
) -> None:
|
|
super().__init__()
|
|
|
|
if output_dim == -1:
|
|
output_dim = input_dim
|
|
|
|
self.norm = LayerNorm(
|
|
input_dim, elementwise_affine=affine,
|
|
device=device, dtype=dtype
|
|
)
|
|
self.proj_in = glu(
|
|
input_dim, hidden_dim, dropout=dropout,
|
|
device=device, dtype=dtype
|
|
)
|
|
self.proj_out = Linear(
|
|
hidden_dim, output_dim, bias=False,
|
|
device=device, dtype=dtype
|
|
)
|
|
|
|
def forward(self, x: Tensor) -> Tensor:
|
|
x = self.norm(x)
|
|
x = self.proj_in(x)
|
|
x = self.proj_out(x)
|
|
return x
|
|
|
|
class _MidBlock(Module):
|
|
def __init__(
|
|
self,
|
|
n_classes: int,
|
|
attn_dim: int,
|
|
head_dim: int,
|
|
ff_dim: int,
|
|
*,
|
|
glu: type[GLU],
|
|
ff_dropout: float,
|
|
device: torch.device | str | None,
|
|
dtype: torch.dtype | None,
|
|
) -> None:
|
|
super().__init__()
|
|
|
|
self.head_dim = head_dim
|
|
|
|
self.q_proj = Linear(
|
|
attn_dim, attn_dim, bias=False,
|
|
device=device, dtype=dtype
|
|
)
|
|
setattr(self.q_proj.weight, "rr_muon_chunks", (1, attn_dim // head_dim))
|
|
|
|
self.q_norm = RMSNorm(head_dim, eps=1e-5, elementwise_affine=False)
|
|
|
|
self.o_proj = Linear(
|
|
attn_dim, attn_dim, bias=False,
|
|
device=device, dtype=dtype
|
|
)
|
|
|
|
self.ff = _FeedForward(
|
|
attn_dim, ff_dim,
|
|
glu=glu,
|
|
dropout=ff_dropout,
|
|
device=device, dtype=dtype,
|
|
)
|
|
|
|
def _forward_q(self, x: Tensor, order: str = "... h s") -> Tensor:
|
|
x = self.q_proj(x)
|
|
x = rearrange(x, f"... s (h e) -> {order} e", e=self.head_dim)
|
|
x = self.q_norm(x)
|
|
return x
|
|
|
|
def _forward_attn(self, x: Tensor, k: Tensor, v: Tensor, attn_mask: Tensor | None) -> Tensor:
|
|
x = self._forward_q(x)
|
|
x = scaled_dot_product_attention(x, k, v, attn_mask=attn_mask)
|
|
x = rearrange(x, "... h s e -> ... s (h e)")
|
|
x = self.o_proj(x)
|
|
return x
|
|
|
|
def _forward_attn_varlen(
|
|
self,
|
|
x: Tensor, k: Tensor, v: Tensor,
|
|
cu_q: Tensor, cu_seq: Tensor, max_q: int, max_seq: int
|
|
) -> Tensor:
|
|
shape = x.shape
|
|
|
|
x = self._forward_q(x, "(... s) h")
|
|
x = cast(Tensor, varlen_attn(x, k, v, cu_q, cu_seq, max_q, max_seq))
|
|
x = x.view(shape)
|
|
x = self.o_proj(x)
|
|
return x
|
|
|
|
def forward(self, x: Tensor, k: Tensor, v: Tensor, attn_mask: Tensor | None) -> Tensor:
|
|
x = x + self._forward_attn(x, k, v, attn_mask)
|
|
x = x + self.ff(x)
|
|
return x
|
|
|
|
def forward_varlen(
|
|
self,
|
|
x: Tensor, k: Tensor, v: Tensor,
|
|
cu_q: Tensor, cu_seq: Tensor, max_q: int, max_seq: int
|
|
) -> Tensor:
|
|
x = x + self._forward_attn_varlen(x, k, v, cu_q, cu_seq, max_q, max_seq)
|
|
x = x + self.ff(x)
|
|
return x
|
|
|
|
class HydraPool(Module):
|
|
def __init__(
|
|
self,
|
|
n_classes: int,
|
|
attn_dim: int,
|
|
head_dim: int,
|
|
*,
|
|
mid_blocks: int = 0,
|
|
input_dim: int = -1,
|
|
ff_dim: int = -1,
|
|
ff_dropout: float = 0.0,
|
|
device: torch.device | str | None = None,
|
|
dtype: torch.dtype | None = None,
|
|
) -> None:
|
|
super().__init__()
|
|
|
|
if input_dim < 0:
|
|
input_dim = attn_dim
|
|
|
|
if ff_dim < 0:
|
|
ff_dim = attn_dim * 3
|
|
|
|
assert attn_dim % head_dim == 0
|
|
n_heads = attn_dim // head_dim
|
|
|
|
self.head_dim = head_dim
|
|
self.output_dim = attn_dim
|
|
|
|
ff_glu: type[GLU]
|
|
if mid_blocks == 0:
|
|
ff_affine = True
|
|
ff_glu = SwiGLU
|
|
else:
|
|
ff_affine = False
|
|
ff_glu = SpGLU
|
|
|
|
self.q = Parameter(torch.randn(
|
|
n_heads, n_classes, head_dim,
|
|
device=device, dtype=dtype
|
|
))
|
|
setattr(self.q, "rr_clsdim", 1)
|
|
|
|
self.kv = Linear(
|
|
input_dim, attn_dim * 2, bias=False,
|
|
device=device, dtype=dtype
|
|
)
|
|
setattr(self.kv.weight, "rr_muon_chunks", (2, n_heads))
|
|
|
|
self.qk_norm = RMSNorm(
|
|
head_dim, eps=1e-5, elementwise_affine=False
|
|
)
|
|
|
|
self.ff = _FeedForward(
|
|
attn_dim, ff_dim,
|
|
glu=ff_glu,
|
|
affine=ff_affine,
|
|
dropout=ff_dropout,
|
|
device=device, dtype=dtype,
|
|
)
|
|
|
|
self.mid_blocks = ModuleList(
|
|
_MidBlock(
|
|
n_classes, attn_dim, head_dim, ff_dim,
|
|
glu=ff_glu,
|
|
ff_dropout=ff_dropout,
|
|
device=device, dtype=dtype
|
|
) for _ in range(mid_blocks)
|
|
)
|
|
|
|
with torch.no_grad():
|
|
self.q.copy_(self.qk_norm(self.q))
|
|
|
|
@property
|
|
def n_classes(self) -> int:
|
|
return self.q.size(1)
|
|
|
|
@property
|
|
def _mid_blocks(self) -> MutableSequence[_MidBlock]:
|
|
return cast(MutableSequence[_MidBlock], self.mid_blocks)
|
|
|
|
def _forward_kv(self, x: Tensor, order: str = "h s") -> tuple[Tensor, Tensor]:
|
|
x = self.kv(x)
|
|
x, v = rearrange(x, f"... s (n h e) -> n ... {order} e", n=2, e=self.head_dim).unbind(0)
|
|
x = self.qk_norm(x)
|
|
return x, v
|
|
|
|
def _forward_attn(self, x: Tensor, attn_mask: Tensor | None) -> tuple[Tensor, Tensor, Tensor]:
|
|
q = self.q.expand(*x.shape[:-2], -1, -1, -1)
|
|
k, v = self._forward_kv(x)
|
|
|
|
x = scaled_dot_product_attention(q, k, v, attn_mask=attn_mask)
|
|
return rearrange(x, "... h s e -> ... s (h e)"), k, v
|
|
|
|
def _forward_attn_varlen(self, x: Tensor, cu_q: Tensor, cu_seq: Tensor, max_seq: int) -> tuple[Tensor, Tensor, Tensor]:
|
|
q = self.q.transpose(0, 1).repeat_interleave(cu_q.size(0) - 1, 0)
|
|
k, v = self._forward_kv(x, "s h")
|
|
|
|
x = cast(Tensor, varlen_attn(q, k, v, cu_q, cu_seq, self.n_classes, max_seq))
|
|
return rearrange(x, "(n s) h e -> n s (h e)", s=self.n_classes), k, v
|
|
|
|
def forward(self, x: Tensor, attn_mask: Tensor | None = None) -> Tensor:
|
|
x, k, v = self._forward_attn(x, attn_mask)
|
|
x = x + self.ff(x)
|
|
|
|
for block in self.mid_blocks:
|
|
x = block(x, k, v, attn_mask)
|
|
|
|
return x
|
|
|
|
def forward_varlen(self, x: Tensor, cu_seq: Tensor, max_seq: int) -> Tensor:
|
|
cu_q = torch.arange(
|
|
0, cu_seq.size(0) * self.n_classes, self.n_classes,
|
|
device=self.q.device, dtype=torch.int32
|
|
)
|
|
|
|
x, k, v = self._forward_attn_varlen(x, cu_q, cu_seq, max_seq)
|
|
x = x + self.ff(x)
|
|
|
|
for block in self._mid_blocks:
|
|
x = block.forward_varlen(x, k, v, cu_q, cu_seq, self.n_classes, max_seq)
|
|
|
|
return x
|
|
|
|
def forward_unpooled(self, x: Tensor, attn_mask: Tensor | None = None) -> Tensor:
|
|
k, v = self._forward_kv(x)
|
|
|
|
x = rearrange(v, "... h s e -> ... s (h e)")
|
|
x = x + self.ff(x)
|
|
|
|
for block in self.mid_blocks:
|
|
x = block(x, k, v, attn_mask)
|
|
|
|
return x
|
|
|
|
def forward_unpooled_varlen(self, x: Tensor, cu_seq: Tensor, max_seq: int) -> Tensor:
|
|
k, v = self._forward_kv(x, "s h")
|
|
|
|
x = rearrange(v, "s h e -> s (h e)")
|
|
x = x + self.ff(x)
|
|
|
|
for block in self._mid_blocks:
|
|
x = block.forward_varlen(x, k, v, cu_seq, cu_seq, max_seq, max_seq)
|
|
|
|
return x
|
|
|
|
@classmethod
|
|
def for_state(
|
|
cls,
|
|
state_dict: dict[str, Any],
|
|
prefix: str = "",
|
|
*,
|
|
num_classes: int | None = None,
|
|
ff_dropout: float = 0.0,
|
|
device: torch.device | str | None = None,
|
|
dtype: torch.dtype | None = None,
|
|
) -> Self:
|
|
n_heads, n_classes, head_dim = state_dict[f"{prefix}q"].shape
|
|
if num_classes is not None:
|
|
n_classes = num_classes
|
|
|
|
attn_dim = n_heads * head_dim
|
|
|
|
input_dim = state_dict[f"{prefix}kv.weight"].size(1)
|
|
ff_dim = state_dict[f"{prefix}ff.proj_out.weight"].size(1)
|
|
|
|
pattern = re.compile(rf"^{re.escape(prefix)}mid_blocks\.([0-9]+)\.")
|
|
mid_blocks = max([-1, *(
|
|
int(match[1])
|
|
for key in state_dict
|
|
if (match := pattern.match(key)) is not None
|
|
)]) + 1
|
|
|
|
return cls(
|
|
n_classes,
|
|
attn_dim,
|
|
head_dim,
|
|
mid_blocks=mid_blocks,
|
|
ff_dim=ff_dim,
|
|
ff_dropout=ff_dropout,
|
|
input_dim=input_dim,
|
|
device=device,
|
|
dtype=dtype
|
|
)
|