🎈 Tokenized.jl

xxxxxxxxxx
 
using DataFrames, CSV, WordTokenizers, Plots, Random, Statistics

18.5 s

Read and Transform the Data

Data source: Shopee - Price Match Guarantee

xxxxxxxxxx

14.5 μs

	title	label_group
	String	Int64
1	"Paper Bag Victoria Secret"	249114794
2	"Double Tape 3M VHB 12 mm x 4,5 m ORIGINAL / DOUBLE FOAM TAPE"	2937985045
3	"Maling TTS Canned Pork Luncheon Meat 397 gr"	2395904891
4	"Daster Batik Lengan pendek - Motif Acak / Campur - Leher Kancing (DPT001-00) Batik karakter Alhadi"	4093212188
5	"Nescafe \\xc3\\x89clair Latte 220ml"	3648931069
6	"CELANA WANITA (BB 45-84 KG)Harem wanita (bisa cod)"	2660605217
7	"Jubah anak size 1-12 thn"	1835033137
8	"KULOT PLISKET SALUR /CANDY PLISKET /WISH KULOT PREMIUM /KULOT PELANGI PREMIUM/HIEKA KULOT"	1565741687
9	"[LOGU] Tempelan kulkas magnet angka, tempelan angka magnet"	2359912463
10	"BIG SALE SEPATU PANTOFEL KULIT KEREN KERJA KANTOR LAKI PRIA COWOK DINAS RESMI FORMAL PESTA KICKERS"	2630990665
more
34250	"FLEX TAPE PELAPIS BOCOR / ISOLASI AJAIB / ANTI BOCOR"	459464107

xxxxxxxxxx
 
df = DataFrame(CSV.File("../data/train.csv"; select=[:title, :label_group]))

15.1 s

Make all characters lower cased:

xxxxxxxxxx

11.3 μs

xxxxxxxxxx
 
df.title = map(x -> lowercase(x), df.title);

44.1 ms

Group by label_group:

xxxxxxxxxx

6.3 μs

groups

GroupedDataFrame with 11014 groups based on key: label_group

First Group (2 rows): label_group = 249114794

	title	label_group
	String	Int64
1	paper bag victoria secret	249114794
2	paper bag victoria secret	249114794

⋮

Last Group (2 rows): label_group = 53836859

	title	label_group
	String	Int64
1	sprei lady rose 180x200 king terlaris keroppi	53836859
2	sprei king ladyrose size 180x200 kerokeroppi	53836859

xxxxxxxxxx
 
groups = groupby(df, "label_group")

137 ms

Tokenizer demo

List of tokenize functions demonstrated here (from the WordTokenizers package):

punctuation_space_tokenize
penn_tokenize
nltk_word_tokenize
poormans_tokenize

xxxxxxxxxx
 
md"## Tokenizer demo
​
List of tokenize functions demonstrated here (from the `WordTokenizers` package):
​
1. `punctuation_space_tokenize`
2. `penn_tokenize`
3. `nltk_word_tokenize`
4. `poormans_tokenize`
"

6.4 μs

SubString{String}1

"sprei"

"lady"

"rose"

"180x200"

"king"

"terlaris"

"keroppi"

xxxxxxxxxx
 
punctuation_space_tokenize("sprei lady rose 180x200 king terlaris keroppi")

57.0 μs

SubString{String}1

"sprei"

"lady"

"rose"

"180x200"

"king"

"terlaris"

"keroppi"

xxxxxxxxxx
 
penn_tokenize("sprei lady rose 180x200 king terlaris keroppi")

219 μs

SubString{String}1

"sprei"

"lady"

"rose"

"180x200"

"king"

"terlaris"

"keroppi"

xxxxxxxxxx
 
punctuation_space_tokenize("sprei lady rose 180x200 king terlaris keroppi")

12.8 μs

String1

"sprei"

"lady"

"rose"

"180"

"x"

"200"

"king"

"terlaris"

"keroppi"

xxxxxxxxxx
 
nltk_word_tokenize("sprei lady rose 180x200 king terlaris keroppi")

23.5 μs

SubString{String}1

"double"

"tape"

"3m"

"vhb"

"12"

"mm"

"x"

"4,5"

"m"

"original"

"double"

"foam"

"tape"

xxxxxxxxxx
 
punctuation_space_tokenize("double tape 3m vhb 12 mm x 4,5 m original / double foam tape")

15.6 μs

SubString{String}1

"double"

"tape"

"3m"

"vhb"

"12"

"mm"

"x"

"45"

"m"

"original"

"double"

"foam"

"tape"

xxxxxxxxxx
 
poormans_tokenize("double tape 3m vhb 12 mm x 4,5 m original / double foam tape")

10.2 μs

SubString{String}1

"double"

"tape"

"3m"

"vhb"

"12"

"mm"

"x"

"4"

","

"5"

"m"

"original"

"/"

"double"

"foam"

"tape"

xxxxxxxxxx
 
penn_tokenize("double tape 3m vhb 12 mm x 4,5 m original / double foam tape")

81.3 μs

String1

"double"

"tape"

"3"

"m"

"vhb"

"12"

"mm"

"x"

"4,5"

"m"

"original"

"/"

"double"

"foam"

"tape"

xxxxxxxxxx
 
nltk_word_tokenize("double tape 3m vhb 12 mm x 4,5 m original / double foam tape")

26.8 μs

Tokenize and Count

xxxxxxxxxx

4.0 μs

tokenize_and_count (generic function with 1 method)

xxxxxxxxxx
 
function tokenize_and_count(groups::GroupedDataFrame, tokenize_func::Function)
    df_combo = DataFrame(
        text_1 = String[], text_2 = String[], label_group = Int64[],
        n_1 = Int64[], n_2 = Int64[], intersect = Int64[], union = Int64[]
    )
    for group in groups
        tokens = map(x -> Set(tokenize_func(x)), group[!, :title])
        for i in 1:(nrow(group)-1)
            for j in (i+1):nrow(group)
                push!(
                    df_combo, 
                    (
                        group[i, :title], group[j, :title], group[i, :label_group],
                        length(tokens[i]), length(tokens[j]),
                        length(∩(tokens[i], tokens[j])),
                        length(∪(tokens[i], tokens[j]))
                    )
                )
            end
        end
    end
    df_combo.jaccard = df_combo.intersect ./ (
        df_combo.n_1 + df_combo.n_2 - df_combo.intersect)
    df_combo.overlap = df_combo.intersect ./ min.(
        df_combo.n_1, df_combo.n_2)
    return df_combo
end

112 μs

Compare results from the two tokenizers (nltk and penn):

xxxxxxxxxx
 
md"Compare results from the two tokenizers (nltk and penn):"

4.8 μs

	text_1	text_2	label_group	n_1	n_2	intersect	union	jaccard	overlap
	String	String	Int64	Int64	Int64	Int64	Int64	Float64	Float64
1	"paper bag victoria secret"	"paper bag victoria secret"	249114794	4	4	4	4	1.0	1.0
2	"double tape 3m vhb 12 mm x 4,5 m original / double foam tape"	"double tape vhb 3m original 12mm x 4.5mm busa perekat"	2937985045	12	12	9	15	0.6	0.75
3	"maling tts canned pork luncheon meat 397 gr"	"maling ham pork luncheon meat tts 397gr"	2395904891	8	8	7	9	0.777778	0.875

xxxxxxxxxx
 
df_nltk = tokenize_and_count(groups, nltk_word_tokenize); df_nltk[1:3, :]

819 ms

	text_1	text_2	label_group	n_1	n_2	intersect	union	jaccard	overlap
	String	String	Int64	Int64	Int64	Int64	Int64	Float64	Float64
1	"paper bag victoria secret"	"paper bag victoria secret"	249114794	4	4	4	4	1.0	1.0
2	"double tape 3m vhb 12 mm x 4,5 m original / double foam tape"	"double tape vhb 3m original 12mm x 4.5mm busa perekat"	2937985045	14	10	6	18	0.333333	0.6
3	"maling tts canned pork luncheon meat 397 gr"	"maling ham pork luncheon meat tts 397gr"	2395904891	8	7	5	10	0.5	0.714286

xxxxxxxxxx
 
df_penn = tokenize_and_count(groups, penn_tokenize); df_penn[1:3, :]

1.2 s

Histogram

xxxxxxxxxx

7.9 μs

xxxxxxxxxx

8.0 ms

xxxxxxxxxx

7.4 ms

xxxxxxxxxx

8.4 ms

xxxxxxxxxx

4.8 ms

xxxxxxxxxx

4.1 ms

xxxxxxxxxx

16.2 ms

xxxxxxxxxx

22.8 ms

xxxxxxxxxx

6.5 ms

(# of groups with fewer than ten members, # of groups with ten or more members):

xxxxxxxxxx

9.7 μs

xxxxxxxxxx

422 ms

Larger groups:

xxxxxxxxxx

7.0 μs

	label_group	nrow
	Int64	Int64
1	1141798720	51
2	3113678103	51
3	562358068	51
4	3627744656	51
5	994676122	51
6	1163569239	51
7	159351600	51

xxxxxxxxxx

199 ms

Pick one group as an example:

xxxxxxxxxx

4.9 μs

	title	label_group
	String	Int64
1	"implora cheek & liptint - implora lip tint original bpom"	3627744656
2	"implora cheek & liptint"	3627744656
3	"lip tint implora cheek & liptint"	3627744656
4	"implora cheek & lip tint model ice cream - set liptint & pemerah pipi"	3627744656
5	"{promo murah} implora cheek&liptint"	3627744656
6	"implora cheek and liptint/ lip tint implora"	3627744656
7	"implora cheek lip tint"	3627744656
8	"new cheek dan liptint/ lip tint by implora bpom"	3627744656
9	"implora cheek & liptint - implora liptint - lip tint implora"	3627744656
10	"new liptint implora / implora cheek & liptint"	3627744656
more
51	"\\xe2\\x9d\\xa4 belia \\xe2\\x9d\\xa4 implora (\\xe2\\x9c\\x94\\xef\\xb8\\x8fbpom) cheek & liptint 5.5g \| lip tint implora"	3627744656

xxxxxxxxxx
 
filter(:label_group => ==(3627744656), df)

12.3 ms

Samples

xxxxxxxxxx

3.7 μs

sample (generic function with 1 method)

xxxxxxxxxx
 
function sample(df::DataFrame, cond, n::Int64)
    df_tmp = filter(cond, df)
    (nrow(df_tmp), df_tmp[shuffle(1:nrow(df_tmp))[1:n], :])
end

22.8 μs

Jaccard index equals 1 (exactly the same):

xxxxxxxxxx

6.7 μs

	text_1	text_2	label_group	n_1	n_2	intersect	union	jaccard	overlap
	String	String	Int64	Int64	Int64	Int64	Int64	Float64	Float64
1	"fanbo all in one deep cleansing balm"	"fanbo all in one deep cleansing balm"	1406531816	7	7	7	7	1.0	1.0
2	"zaitun palestin 100 kapsul \| extra virgin oil"	"zaitun palestin 100 kapsul \| extra virgin oil"	1503360084	8	8	8	8	1.0	1.0
3	"foil balon angka 0-916in 40cm balloon pesta foil warna gold silver ulang tahun happy birthday"	"foil balon angka 0-916in 40cm balloon pesta foil warna gold silver ulang tahun happy birthday"	2196920644	18	18	18	18	1.0	1.0
4	"rsb kotak jam persegi box watch kotak arloji kotak gelang kotak perhiasan kotak hadiah sourvenir"	"rsb kotak jam persegi box watch kotak arloji kotak gelang kotak perhiasan kotak hadiah sourvenir"	2968086582	11	11	11	11	1.0	1.0
5	"madame gie color pop lip balm"	"madame gie color pop lip balm"	2339128108	6	6	6	6	1.0	1.0
6	"jilbab anak elmo"	"jilbab anak elmo"	3149716162	3	3	3	3	1.0	1.0
7	"dr pure cream set 3in1 original bpom"	"dr pure cream set 3in1 original bpom"	4226111444	9	9	9	9	1.0	1.0
8	"make over riche glow face highlighter"	"make over riche glow face highlighter"	338620322	6	6	6	6	1.0	1.0
9	"pisau apel"	"pisau apel"	952183003	2	2	2	2	1.0	1.0
10	"kanebo refil polos sedang - grosir - aion plas chamois mobil motor kain lap microfiber"	"kanebo refil polos sedang - grosir - aion plas chamois mobil motor kain lap microfiber"	821583868	14	14	14	14	1.0	1.0

xxxxxxxxxx
 
sample(df_nltk, :jaccard => x -> x == 1, 10)

388 μs

Jaccard index equals 0 (completely different):

xxxxxxxxxx

4.7 μs

	text_1	text_2	label_group	n_1	n_2	intersect	union	jaccard	overlap
	String	String	Int64	Int64	Int64	Int64	Int64	Float64	Float64
1	"sunday simpel dress"	"( 1kg = 5pcs ) reinn gamis busui resleting depan bsw monalisa maxmara"	2133079047	3	15	0	18	0.0	0.0
2	"lanbena masker peel off untuk menghilangkan komedo hitam"	"breylee blackhead mask (step 1) / pore serum (step 2)"	3001123709	8	11	0	19	0.0	0.0
3	"cutbray kirien kulot.bahan scuba.onesize.pjg_+90.cm. lp_+58.s/d.105.cm.bahan melar pingang karet"	"premium jilbab segi 3 segitiga diamond soft pad antem bobal (bolak balik)"	2008989859	18	14	0	32	0.0	0.0
4	"zr-208"	"zara suede vinyl mule heels"	622201978	2	5	0	7	0.0	0.0
5	"ready stock w dressroom wdressroom new york 97 april cotton jungkook 45 morning rain v taehyung bts"	"w.dressroom dress & living season 2 clear perfume 70ml/150ml"	3284713135	17	12	0	29	0.0	0.0
6	"hijab khadijah by anaya hijab"	"jilbab instan madina niqab jersey - kerudung wanita"	3768186621	4	8	0	12	0.0	0.0
7	"balet inaya ikat tengah aa - iib"	"unique flat shoes karina ribbon fs37 hitam maroon mocca abu"	2458827579	7	11	0	18	0.0	0.0
8	"rasha_sepatu adidas alphabounce terbaru size 36-44.. bisa cod..!!!"	"sepatupria/sepatualphabounce/sepaturuning"	3027068648	14	4	0	18	0.0	0.0
9	"belt fashion"	"ikat pinggang strap kepang tipis bahan jalinan tambang"	1003252496	2	8	0	10	0.0	0.0
10	"(1kg muat 6pcs) cozy kulot pants"	"jilbab/hijab segi 3 segitiga soft pad bobal khimar 2 layer"	2008989859	10	12	0	22	0.0	0.0

xxxxxxxxxx
 
sample(df_nltk, :jaccard => x -> x == 0, 10)

217 μs

Jaccard index between 0 and 0.2 (only slightly similar):

xxxxxxxxxx

5.1 μs

	text_1	text_2	label_group	n_1	n_2	intersect	union	jaccard	overlap
	String	String	Int64	Int64	Int64	Int64	Int64	Float64	Float64
1	"minyak kemiri premium al-qodry \| penumbuh rambut \| penghitam rambut 125ml"	"minyak kemiri al khodry original"	3000792516	10	5	2	13	0.153846	0.4
2	"original cuddle me adjustable training pants / celana dalam latihan pipis anak"	"[bisa cod!] \\xe2\\x80\\xa2 training pants cuddleme\\xe2\\x80\\xa2 celana belajar pipis/tatur bayi balita (+adjustable waist)"	2999380171	12	25	5	32	0.15625	0.416667
3	"jam tangan wanita analog garansi harga termurah jam tangan wanita fashion"	"grosir free baterai jam tangan casio g-shock baby-g bga-110 b666 rubber import grosir gk024"	777596345	8	16	2	22	0.0909091	0.25
4	"pota sweet potato chips brown sugar balado 50g"	"barefood keripik ubi pota 50gr / barefood keripik pisang bana"	1045293852	9	9	2	16	0.125	0.222222
5	"tas selempang wanita rumbay fierly"	"tas selempang wanita sophie fairiese t3036p3 esmonia lopper hardware gold original seri e1416 u2r2"	2418158256	5	20	3	22	0.136364	0.6
6	"[ready stock] ikat rambut korea simple cute murah"	"100 pcs ikat rambut karet polos elastis gaya korea untuk review wanita"	994676122	10	12	3	19	0.157895	0.3
7	"gamis anak perempuan (bayar di tempat) 5-11th size m l leticia saleha series lebaran all white"	"leticia pakaian anak perempuan plus hijab korean fashion sleeveless lace floral gaun usia 5-7thn"	1274738046	21	17	5	33	0.151515	0.294118
8	"gamis rayon slavina leestari 06"	"gamis wolfis sleting batiq 108"	2087590585	5	5	1	9	0.111111	0.2
9	"[ultra color] garnier color natural ultra sachet / garnier pewarna rambut"	"garnier color naturals express creme 3.16 - merah burgundy"	2910074820	10	9	2	17	0.117647	0.222222
10	"tetes softlens a+ 10ml"	"[import] - tetes mata/ obat tetes cairan softles a + 10ml"	3323080982	5	14	3	16	0.1875	0.6

xxxxxxxxxx
 
sample(df_nltk, :jaccard => x -> (x > 0) & (x < 0.2), 10)

682 μs