```python
from transformers import AlbertTokenizer, AlbertModel, AlbertForMaskedLM,AlbertConfig, AlbertForPreTraining
import torch

config = AlbertConfig.from_json_file('model/config.json')
model =  AlbertForPreTraining.from_pretrained('model/pytorch_model.bin', config=config)
tokenizer = AlbertTokenizer("model/sentencepiece.model")
```

    Some weights of AlbertForPreTraining were not initialized from the model checkpoint at model/pytorch_model.bin and are newly initialized: ['albert.pooler.weight', 'albert.pooler.bias', 'sop_classifier.classifier.weight', 'sop_classifier.classifier.bias']
    You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.



```python
import pandas as pd
df = pd.read_csv("data/pair.csv",names=["sentence1","sentence2"],header=None)
df = df.iloc[1:]
df["sop_label"] = 0
```


```python
df
```


```python
df1 = pd.DataFrame()
df1["sentence1"] = df["sentence2"]
df1["sentence2"] = df["sentence1"]
df1["sop_label"] = 1
df1.head()
```


```python
df2 = pd.concat([df,df1])
df2 = df2.sample(frac=1)
df2.head()
```


```python
df2 = df2.dropna()
```


```python
df2.to_csv("sop_sen_pair.csv")
```


```python
from transformers import DataCollatorForLanguageModeling
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer, mlm=True, mlm_probability=0.15
)
```


```python
tokenizer([("＜目的＞XXの満開50日後の果実肥大調査。" ,"平年並み-やや大きい。"),("営農センターに在庫があるので、出荷の際とりにくるとのこと。","水稲肥料の一発かんたくんを6袋受注する。")]
             ,padding = "max_length",max_length = 100)
```


```python
tokenizer.convert_ids_to_tokens(tokenizer("＜目的＞ももの満開50日後の果実肥大調査。" ,"平年並み-やや大きい。")["input_ids"])
```


```python
from transformers import DataCollatorForLanguageModeling
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer, mlm=True, mlm_probability=0.15
)
```


```python
data_collator(tokenizer([("＜目的＞ももの満開50日後の果実肥大調査。" ,"平年並み-やや大きい。")]
             ,padding = "max_length",max_length = 100)["input_ids"])
```


```python
import pandas as pd
df2 = pd.read_csv("data/sop_sen_pair.csv")
```


```python
df2.columns
```




    Index(['Unnamed: 0', 'sentence1', 'sentence2', 'sop_label'], dtype='object')




```python
df2 = df2.dropna()[["sentence1","sentence2","sop_label"]]
df2.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>sentence1</th>
      <th>sentence2</th>
      <th>sop_label</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>＜目的＞XXの満開50日後の果実肥大調査。</td>
      <td>平年並み-やや大きい。</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>＜目的＞ゼスプリゴールドギャップ内部監査＜結果＞昨年監査に当たっている為比較的資料はそろっ...</td>
      <td>＜対応＞資料がそろっているか確認する。</td>
      <td>0</td>
    </tr>
    <tr>
      <th>2</th>
      <td>営農センターに在庫があるので、出荷の際とりにくるとのこと。</td>
      <td>水稲肥料の一発かんたくんを6袋受注する。</td>
      <td>0</td>
    </tr>
    <tr>
      <th>3</th>
      <td>自動車、コンロ、農機についての推進を行った。</td>
      <td>＜目的＞高温に係る異常早期警戒情報についてと推進活動＜結果＞今月いっぱいXX側を中心に気温...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>4</th>
      <td>虫ではダニが多く見られたため殺虫を指示。</td>
      <td>ダニトロン（ダニ）早急に肥料を効かせるため液肥と1％硫マグ溶液の両面散布を勧める。</td>
      <td>0</td>
    </tr>
  </tbody>
</table>
</div>




```python
from datasets import Dataset
import pandas as pd
dataset = Dataset.from_pandas(df2)
```


```python
dataset[0]
```




    {'sentence1': ' ＜目的＞ももの満開50日後の果実肥大調査。',
     'sentence2': '平年並み-やや大きい。',
     'sop_label': 0,
     '__index_level_0__': 0}




```python

```


```python
import tqdm
input_ids_list = []
token_type_ids = []
attention_mask = []
for index, row in tqdm.tqdm(df2.iterrows()):
    try:
        tokens = tokenizer(row.sentence1,row.sentence2,padding = "max_length",max_length = 100)
        input_ids_list.append(tokens["input_ids"])
        token_type_ids.append(tokens["token_type_ids"])
        attention_mask.append(tokens["attention_mask"])
    except:
        print(row)
```

    717058it [05:30, 2170.45it/s]



```python
df2["input_ids"] = input_ids_list
df2["token_type_ids"] = token_type_ids
df2["attention_mask"] = attention_mask
```

   
    
    See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      df2["token_type_ids"] = token_type_ids
    <ipython-input-11-9ccbad34920b>:3: SettingWithCopyWarning: 
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead
    
    See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      df2["attention_mask"] = attention_mask



```python
df2.to_csv("tokened_2seq.csv")
```


```python
from transformers import AlbertTokenizer, AlbertModel, AlbertForMaskedLM,AlbertConfig, Alb

Sentence order predict

Discussion

コメントにはログインが必要です。