sprx7767 commited on
Commit
f048237
·
verified ·
1 Parent(s): 8a630fd

feat: enable CNN Auto loading

Browse files
Files changed (5) hide show
  1. cnn/README.md +10 -28
  2. cnn/config.json +23 -0
  3. cnn/hf_model.py +176 -0
  4. cnn/manifest.json +3 -1
  5. cnn/tokenizer_config.json +11 -0
cnn/README.md CHANGED
@@ -1,36 +1,18 @@
1
  ---
 
2
  license: other
3
  license_name: chandassu-citation
4
  license_link: LICENSE
5
- language: te
6
- tags: [telugu, poetry, metre, cnn]
7
- library_name: pytorch
8
  pipeline_tag: text-classification
 
9
  ---
10
  # Chandassu CNN
11
 
12
- Four-class Telugu poetic metre classifier, trained on NVIDIA H100 NVL.
13
- Seed 17; selected epoch 59; 68,420 parameters.
14
- Calibration temperature: 1.34592281013039. Mean calibrated line probabilities classify a four-line poem.
15
-
16
- Classes: ఉత్పలమాల, చంపకమాల, మత్తేభము, శార్దూలం (in this order).
17
-
18
- Load this package with `chandassu.artifacts.load_package`, or run the repository's
19
- `inference.py --model_dir PATH`. This custom encoder/head package does not implement
20
- the generic Transformers AutoModel interface. No pickle is required for inference.
21
-
22
- Training: 6,517 poems; separate calibration: 908; selection: 1,859.
23
- Historical diagnostic: 2,234 poems, already inspected during earlier project development.
24
- See metadata.json and the repository results/v1 for full metrics and provenance.
25
- This is a restricted classifier: it cannot reliably reject other metres or provide
26
- prosodic explanations. Scores are calibrated probabilities, not confidence intervals.
27
- Source edition/transcription rights are not independently cleared; dataset terms
28
- retain their source terms. See the repository data/LICENSE.md.
29
- Project-owned material uses the citation terms in LICENSE. Earlier Apache-2.0 permissions remain in effect.
30
-
31
- ## Citation
32
-
33
- Pranav Surampudi (2026). Chandassu Recognition: Telugu Poetic Meter Classification.
34
- https://github.com/Pranav-20186017/chandassu_recognition
35
-
36
- See CITATION.cff for citation metadata and LICENSE for the applicable terms.
 
1
  ---
2
+ language: te
3
  license: other
4
  license_name: chandassu-citation
5
  license_link: LICENSE
6
+ library_name: transformers
 
 
7
  pipeline_tag: text-classification
8
+ tags: [telugu, poetry, metre, cnn]
9
  ---
10
  # Chandassu CNN
11
 
12
+ Use `AutoTokenizer` and `AutoModelForSequenceClassification` with
13
+ `subfolder="cnn"` and `trust_remote_code=True`.
14
+ The model returns raw logits for each line. Divide logits by `model.config.calibration_temperature`
15
+ before applying softmax. For a four-line poem, average the four probability vectors.
16
+ See the [main model card](https://huggingface.co/sprx7767/chandassu-recognition)
17
+ for code, results, limits, and citation terms.
18
+ The project loader and `inference.py` also support this package.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cnn/config.json CHANGED
@@ -1,7 +1,16 @@
1
  {
 
 
 
2
  "artifact_schema": 1,
 
 
 
 
 
3
  "base_model": "google/byt5-small",
4
  "calibration_fraction": 0.15,
 
5
  "cpu_threads": 4,
6
  "data_dir": "data/v1",
7
  "device": "auto",
@@ -13,12 +22,24 @@
13
  "folds": 3,
14
  "gradient_checkpointing": false,
15
  "gradient_clip": 1.0,
 
 
 
 
 
 
16
  "init_checkpoint": null,
17
  "kernel_sizes": [
18
  3,
19
  5,
20
  7
21
  ],
 
 
 
 
 
 
22
  "labels": [
23
  "ఉత్పలమాల",
24
  "చంపకమాల",
@@ -37,6 +58,7 @@
37
  "model_revision": "68377bdc18a2ffec8a0533fef03b1c513a4dd49d",
38
  "model_type": "cnn",
39
  "output_dir": "runs/cnn-seed-17",
 
40
  "patience": 12,
41
  "precision": "auto",
42
  "progress_every": 50,
@@ -46,6 +68,7 @@
46
  "split_manifest": "splits/final_h100.json",
47
  "split_seed": 142,
48
  "temperature": 1.34592281013039,
 
49
  "weight_decay": 0.01,
50
  "width": 64
51
  }
 
1
  {
2
+ "architectures": [
3
+ "ChandassuCNNForSequenceClassification"
4
+ ],
5
  "artifact_schema": 1,
6
+ "auto_map": {
7
+ "AutoConfig": "hf_model.ChandassuCNNConfig",
8
+ "AutoModel": "hf_model.ChandassuCNNForSequenceClassification",
9
+ "AutoModelForSequenceClassification": "hf_model.ChandassuCNNForSequenceClassification"
10
+ },
11
  "base_model": "google/byt5-small",
12
  "calibration_fraction": 0.15,
13
+ "calibration_temperature": 1.34592281013039,
14
  "cpu_threads": 4,
15
  "data_dir": "data/v1",
16
  "device": "auto",
 
22
  "folds": 3,
23
  "gradient_checkpointing": false,
24
  "gradient_clip": 1.0,
25
+ "id2label": {
26
+ "0": "ఉత్పలమాల",
27
+ "1": "చంపకమాల",
28
+ "2": "మత్తేభము",
29
+ "3": "శార్దూలం"
30
+ },
31
  "init_checkpoint": null,
32
  "kernel_sizes": [
33
  3,
34
  5,
35
  7
36
  ],
37
+ "label2id": {
38
+ "ఉత్పలమాల": 0,
39
+ "చంపకమాల": 1,
40
+ "మత్తేభము": 2,
41
+ "శార్దూలం": 3
42
+ },
43
  "labels": [
44
  "ఉత్పలమాల",
45
  "చంపకమాల",
 
58
  "model_revision": "68377bdc18a2ffec8a0533fef03b1c513a4dd49d",
59
  "model_type": "cnn",
60
  "output_dir": "runs/cnn-seed-17",
61
+ "pad_token_id": 0,
62
  "patience": 12,
63
  "precision": "auto",
64
  "progress_every": 50,
 
68
  "split_manifest": "splits/final_h100.json",
69
  "split_seed": 142,
70
  "temperature": 1.34592281013039,
71
+ "vocab_size": 94,
72
  "weight_decay": 0.01,
73
  "width": 64
74
  }
cnn/hf_model.py ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Standalone Transformers classes exported with Chandassu model packages.
2
+
3
+ Copyright 2026 Pranav Surampudi. SPDX-License-Identifier: LicenseRef-Chandassu-Citation
4
+ This module intentionally depends only on PyTorch, Transformers and the standard library.
5
+ """
6
+
7
+ import json
8
+ import shutil
9
+ import unicodedata
10
+ from pathlib import Path
11
+
12
+ import torch
13
+ from torch import nn
14
+ from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizer, T5Config, T5EncoderModel
15
+ from transformers.modeling_outputs import SequenceClassifierOutput
16
+
17
+
18
+ class ChandassuConfig(PretrainedConfig):
19
+ """Keep the original fitting configuration and calibrated temperature."""
20
+
21
+ def __init__(self, labels=None, temperature=1.0, calibration_temperature=None, **kwargs):
22
+ labels = labels or ["ఉత్పలమాల", "చంపకమాల", "మత్తేభము", "శార్దూలం"]
23
+ kwargs.setdefault("id2label", dict(enumerate(labels)))
24
+ kwargs.setdefault("label2id", {label: i for i, label in enumerate(labels)})
25
+ super().__init__(**kwargs)
26
+ self.labels = labels
27
+ self.calibration_temperature = temperature if calibration_temperature is None else calibration_temperature
28
+ if len(labels) != 4 or not 0 < self.calibration_temperature < float("inf"):
29
+ raise ValueError("Require four labels and a finite positive temperature")
30
+
31
+
32
+ class ChandassuCNNConfig(ChandassuConfig):
33
+ model_type = "cnn"
34
+
35
+
36
+ class ChandassuByT5Config(ChandassuConfig):
37
+ model_type = "byt5"
38
+
39
+
40
+ class ChandassuTokenizer(PreTrainedTokenizer):
41
+ """Exact training-time NFC, whitespace, character and UTF-8 byte encoding."""
42
+
43
+ vocab_files_names = {"vocabulary_file": "vocabulary.json"}
44
+ model_input_names = ["input_ids", "attention_mask"]
45
+
46
+ def __init__(self, vocabulary_file, **kwargs):
47
+ self.vocabulary_file = str(vocabulary_file)
48
+ saved = json.loads(Path(vocabulary_file).read_text())
49
+ self.kind = saved["kind"]
50
+ self.limit = saved["max_tokens"]
51
+ if self.kind == "character":
52
+ self.tokens = {"[PAD]": 0, "[UNK]": 1, **saved["tokens"]}
53
+ defaults = {"pad_token": "[PAD]", "unk_token": "[UNK]"}
54
+ elif self.kind == "utf8_bytes":
55
+ self.tokens = {"<pad>": 0, "</s>": 1, "<unk>": 2}
56
+ self.tokens.update({f"<byte:{i}>": i + 3 for i in range(256)})
57
+ defaults = {"pad_token": "<pad>", "eos_token": "</s>", "unk_token": "<unk>"}
58
+ else:
59
+ raise ValueError("Unsupported vocabulary kind")
60
+ self.reverse_tokens = {value: key for key, value in self.tokens.items()}
61
+ for key, value in defaults.items():
62
+ kwargs.setdefault(key, value)
63
+ kwargs.setdefault("model_max_length", self.limit)
64
+ kwargs.setdefault("padding_side", "right")
65
+ super().__init__(**kwargs)
66
+
67
+ @property
68
+ def vocab_size(self):
69
+ return len(self.tokens)
70
+
71
+ def get_vocab(self):
72
+ return dict(self.tokens, **self.added_tokens_encoder)
73
+
74
+ def tokenize(self, text, **kwargs):
75
+ # Override special-token splitting: literal text must have the same IDs as training.
76
+ text = " ".join(unicodedata.normalize("NFC", text).replace("\ufeff", "").split())
77
+ if self.kind == "character":
78
+ if not text or len(text) > self.limit:
79
+ raise ValueError(f"Line length outside 1..{self.limit}; no truncation")
80
+ return list(text)
81
+ encoded = text.encode("utf-8")
82
+ if len(encoded) + 1 > self.limit:
83
+ raise ValueError("Byte line exceeds capacity; no truncation")
84
+ return [f"<byte:{value}>" for value in encoded]
85
+
86
+ def _tokenize(self, text):
87
+ return self.tokenize(text)
88
+
89
+ def _convert_token_to_id(self, token):
90
+ return self.tokens.get(token, self.tokens[str(self.unk_token)])
91
+
92
+ def _convert_id_to_token(self, index):
93
+ return self.reverse_tokens.get(index, str(self.unk_token))
94
+
95
+ def convert_tokens_to_string(self, tokens):
96
+ if self.kind == "character":
97
+ return "".join(tokens)
98
+ return bytes(int(t[6:-1]) for t in tokens if t.startswith("<byte:")).decode("utf-8", errors="replace")
99
+
100
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
101
+ if token_ids_1 is not None:
102
+ raise ValueError("Encode poem lines separately; text pairs are unsupported")
103
+ return token_ids_0 + ([1] if self.kind == "utf8_bytes" else [])
104
+
105
+ def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False):
106
+ if already_has_special_tokens:
107
+ return super().get_special_tokens_mask(token_ids_0, token_ids_1, True)
108
+ return [0] * len(token_ids_0) + ([1] if self.kind == "utf8_bytes" else [])
109
+
110
+ def prepare_for_model(self, *args, **kwargs):
111
+ if kwargs.get("truncation") not in (None, False, "do_not_truncate"):
112
+ raise ValueError("Truncation is unsupported; supply a complete line within the input limit")
113
+ return super().prepare_for_model(*args, **kwargs)
114
+
115
+ def save_vocabulary(self, save_directory, filename_prefix=None):
116
+ name = (filename_prefix + "-" if filename_prefix else "") + "vocabulary.json"
117
+ destination = Path(save_directory) / name
118
+ if destination.resolve() != Path(self.vocabulary_file).resolve():
119
+ shutil.copyfile(self.vocabulary_file, destination)
120
+ return (str(destination),)
121
+
122
+
123
+ class ChandassuClassifier(PreTrainedModel):
124
+ """Four raw line logits; use config.calibration_temperature for calibrated probabilities."""
125
+
126
+ config_class = ChandassuConfig
127
+ _tied_weights_keys = ["encoder.encoder.embed_tokens.weight"]
128
+
129
+ def __init__(self, config):
130
+ super().__init__(config)
131
+ self.dropout = nn.Dropout(config.dropout)
132
+ if config.model_type == "cnn":
133
+ self.embedding = nn.Embedding(config.vocab_size, config.width, padding_idx=0)
134
+ self.convolutions = nn.ModuleList(
135
+ [nn.Conv1d(config.width, config.width, k, padding=k // 2) for k in config.kernel_sizes]
136
+ )
137
+ self.classifier = nn.Linear(config.width * len(config.kernel_sizes), 4)
138
+ elif config.model_type == "byt5":
139
+ self.encoder = T5EncoderModel(T5Config.from_dict(config.encoder_config))
140
+ self.classifier = nn.Linear(self.encoder.config.d_model, 4)
141
+ else:
142
+ raise ValueError("Unsupported model family")
143
+ self.post_init()
144
+
145
+ def forward(self, input_ids, attention_mask=None, labels=None, return_dict=None, **kwargs):
146
+ valid = input_ids.ne(0)
147
+ if not valid.any(1).all():
148
+ raise ValueError("Empty line")
149
+ if attention_mask is not None and not torch.equal(attention_mask.bool(), valid):
150
+ raise ValueError("Attention mask must match non-padding token IDs")
151
+ if self.config.model_type == "cnn":
152
+ embedded = self.embedding(input_ids).masked_fill(~valid.unsqueeze(-1), 0.0)
153
+ features = self.dropout(embedded).transpose(1, 2)
154
+ pooled = torch.cat(
155
+ [
156
+ torch.relu(conv(features)).masked_fill(~valid.unsqueeze(1), float("-inf")).amax(2)
157
+ for conv in self.convolutions
158
+ ],
159
+ dim=1,
160
+ )
161
+ else:
162
+ hidden = self.encoder(input_ids=input_ids, attention_mask=valid.long(), return_dict=True).last_hidden_state
163
+ pooled = (hidden * valid.unsqueeze(-1)).sum(1) / valid.sum(1, keepdim=True)
164
+ logits = self.classifier(self.dropout(pooled))
165
+ loss = nn.functional.cross_entropy(logits, labels) if labels is not None else None
166
+ if return_dict is False or (return_dict is None and not self.config.use_return_dict):
167
+ return (loss, logits) if loss is not None else (logits,)
168
+ return SequenceClassifierOutput(loss=loss, logits=logits)
169
+
170
+
171
+ class ChandassuCNNForSequenceClassification(ChandassuClassifier):
172
+ config_class = ChandassuCNNConfig
173
+
174
+
175
+ class ChandassuByT5ForSequenceClassification(ChandassuClassifier):
176
+ config_class = ChandassuByT5Config
cnn/manifest.json CHANGED
@@ -1,8 +1,10 @@
1
  {
2
  "files": {
3
- "config.json": "1a7134e4e792ee671344d16268890462a608af42df12b24e6240e6324bb4d25e",
 
4
  "metadata.json": "550c82d89c0cc8607c90f6b7ae5af99bd1ec62ab48754d84808c62258a42d731",
5
  "model.safetensors": "6d8d642f8b83343e6e98c90365b1449acf0fcedf430f4b4e727a01979aa4e95e",
 
6
  "vocabulary.json": "6d3991cf7463459f411f6e3bc03bc5f14b4274a712e6853d784c00835ce14cee"
7
  },
8
  "schema_version": 1
 
1
  {
2
  "files": {
3
+ "config.json": "fc5f10b725dfbb4e081f7789740361f14daa12eede4827cafff2517e442fc37a",
4
+ "hf_model.py": "d9b75c12812f843a8b4f95f275add3c946773f07a65c24d46be08f55edceb5e3",
5
  "metadata.json": "550c82d89c0cc8607c90f6b7ae5af99bd1ec62ab48754d84808c62258a42d731",
6
  "model.safetensors": "6d8d642f8b83343e6e98c90365b1449acf0fcedf430f4b4e727a01979aa4e95e",
7
+ "tokenizer_config.json": "3ee891d67d5c1c0b7c6eb08e537ab973c9dc77eed067ee64ea36c59ba7a1e91a",
8
  "vocabulary.json": "6d3991cf7463459f411f6e3bc03bc5f14b4274a712e6853d784c00835ce14cee"
9
  },
10
  "schema_version": 1
cnn/tokenizer_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoTokenizer": [
4
+ "hf_model.ChandassuTokenizer",
5
+ null
6
+ ]
7
+ },
8
+ "model_max_length": 256,
9
+ "padding_side": "right",
10
+ "tokenizer_class": "ChandassuTokenizer"
11
+ }