blob: 4f84469ef15c443abd662a0938291f01243e70d0 (plain)

ofs	hex dump	ascii
0000	b3 f2 0d 0a a1 6c 87 4a 63 00 00 00 00 00 00 00 00 1a 00 00 00 40 00 00 00 73 a6 04 00 00 64 00	.....l.Jc............@...s....d.
0020	00 5a 00 00 64 01 00 5a 01 00 64 02 00 5a 02 00 64 03 00 64 04 00 6b 03 00 5a 03 00 64 03 00 64	.Z..d..Z..d..Z..d..d..k..Z..d..d
0040	04 00 6b 04 00 5a 04 00 64 03 00 64 05 00 6b 05 00 54 64 03 00 64 04 00 6b 05 00 5a 05 00 67 00	..k..Z..d..d..k..Td..d..k..Z..g.
0060	00 04 5a 06 00 65 07 00 65 05 00 83 01 00 44 5d 22 00 5a 08 00 65 08 00 64 06 00 19 64 07 00 6a	..Z..e..e.....D]".Z..e..d...d..j
0080	03 00 6f 0b 00 01 65 06 00 65 08 00 12 71 51 00 01 71 51 00 5b 06 00 64 08 00 64 09 00 64 0a 00	..o...e..e...qQ..qQ.[..d..d..d..
00a0	64 0b 00 64 0c 00 67 05 00 17 5a 09 00 5b 08 00 5b 05 00 65 0a 00 5a 0b 00 64 08 00 65 0c 00 65	d..d..g...Z..[..[..e..Z..d..e..e
00c0	0b 00 3c 65 0a 00 64 0d 00 17 5a 0d 00 64 0b 00 65 0c 00 65 0d 00 3c 65 0a 00 64 0e 00 37 5a 0a	..<e..d...Z..d..e..e..<e..d..7Z.
00e0	00 64 0f 00 84 00 00 5a 0e 00 64 10 00 84 00 00 5a 0f 00 64 11 00 84 00 00 5a 10 00 64 12 00 5a	.d.....Z..d.....Z..d.....Z..d..Z
0100	11 00 64 13 00 5a 12 00 65 11 00 65 0f 00 64 14 00 65 11 00 17 83 01 00 17 65 10 00 65 12 00 83	..d..Z..e..e..d..e.......e..e...
0120	01 00 17 5a 13 00 64 15 00 5a 14 00 64 16 00 5a 15 00 64 17 00 5a 16 00 64 18 00 5a 17 00 65 0e	...Z..d..Z..d..Z..d..Z..d..Z..e.
0140	00 65 15 00 65 16 00 65 17 00 83 03 00 5a 18 00 64 19 00 5a 19 00 65 0e 00 64 1a 00 64 1b 00 83	.e..e..e.....Z..d..Z..e..d..d...
0160	02 00 65 10 00 65 19 00 83 01 00 17 5a 1a 00 64 1c 00 65 19 00 17 5a 1b 00 65 0e 00 65 1a 00 65	..e..e......Z..d..e...Z..e..e..e
0180	1b 00 83 02 00 5a 1c 00 65 0e 00 64 1d 00 65 1c 00 64 1e 00 17 83 02 00 5a 1d 00 65 0e 00 65 1d	.....Z..e..d..e..d......Z..e..e.
01a0	00 65 1c 00 65 18 00 83 03 00 5a 1e 00 64 1f 00 5a 1f 00 64 20 00 5a 20 00 64 21 00 5a 21 00 64	.e..e.....Z..d..Z..d..Z..d!.Z!.d
01c0	22 00 5a 22 00 65 0e 00 64 23 00 64 24 00 83 02 00 5a 23 00 65 0e 00 64 25 00 64 26 00 83 02 00	".Z".e..d#.d$....Z#.e..d%.d&....
01e0	5a 24 00 65 0e 00 64 27 00 64 28 00 64 29 00 64 2a 00 64 2b 00 64 2c 00 64 2d 00 64 2e 00 83 08	Z$.e..d'.d(.d).d*.d+.d,.d-.d....
0200	00 5a 25 00 64 2f 00 5a 26 00 65 0e 00 64 30 00 64 31 00 83 02 00 5a 27 00 65 0e 00 65 25 00 65	.Z%.d/.Z&.e..d0.d1....Z'.e..e%.e
0220	26 00 65 27 00 83 03 00 5a 28 00 65 0e 00 65 1e 00 65 28 00 65 24 00 65 14 00 83 04 00 5a 29 00	&.e'....Z(.e..e..e(.e$.e.....Z).
0240	65 13 00 65 29 00 17 5a 2a 00 65 0e 00 64 32 00 65 0e 00 64 33 00 64 14 00 83 02 00 17 64 34 00	e..e)..Z*.e..d2.e..d3.d......d4.
0260	65 0e 00 64 35 00 64 14 00 83 02 00 17 83 02 00 5a 2b 00 65 0e 00 64 14 00 65 12 00 65 23 00 83	e..d5.d.........Z+.e..d..e..e#..
0280	03 00 5a 2c 00 65 11 00 65 0e 00 65 2c 00 65 1e 00 65 28 00 65 2b 00 65 14 00 83 05 00 17 5a 2d	..Z,.e..e..e,.e..e(.e+.e......Z-
02a0	00 65 2e 00 65 04 00 69 2f 00 65 2a 00 65 2d 00 65 21 00 65 22 00 66 04 00 83 02 00 5c 04 00 5a	.e..e..i/.e*.e-.e!.e".f.....\..Z
02c0	30 00 5a 31 00 5a 32 00 5a 33 00 68 00 00 04 65 04 00 69 2f 00 65 1f 00 83 01 00 02 64 33 00 3c	0.Z1.Z2.Z3.h...e..i/.e......d3.<
02e0	04 65 04 00 69 2f 00 65 20 00 83 01 00 02 64 35 00 3c 04 65 32 00 02 64 36 00 3c 04 65 33 00 02	.e..i/.e......d5.<.e2..d6.<.e3..
0300	64 37 00 3c 04 65 32 00 02 64 38 00 3c 04 65 33 00 02 64 39 00 3c 04 65 32 00 02 64 3a 00 3c 04	d7.<.e2..d8.<.e3..d9.<.e2..d:.<.
0320	65 33 00 02 64 3b 00 3c 04 65 32 00 02 64 3c 00 3c 04 65 33 00 02 64 3d 00 3c 04 65 32 00 02 64	e3..d;.<.e2..d<.<.e3..d=.<.e2..d
0340	3e 00 3c 04 65 33 00 02 64 3f 00 3c 04 65 32 00 02 64 40 00 3c 04 65 33 00 02 64 41 00 3c 04 65	>.<.e3..d?.<.e2..d@.<.e3..dA.<.e
0360	32 00 02 64 42 00 3c 04 65 33 00 02 64 43 00 3c 04 65 32 00 02 64 44 00 3c 04 65 33 00 02 64 45	2..dB.<.e3..dC.<.e2..dD.<.e3..dE
0380	00 3c 04 65 32 00 02 64 46 00 3c 04 65 33 00 02 64 47 00 3c 04 64 04 00 02 64 48 00 3c 04 64 04	.<.e2..dF.<.e3..dG.<.d...dH.<.d.
03a0	00 02 64 49 00 3c 04 64 04 00 02 64 4a 00 3c 04 64 04 00 02 64 4b 00 3c 5a 35 00 68 00 00 5a 36	..dI.<.d...dJ.<.d...dK.<Z5.h..Z6
03c0	00 78 18 00 64 67 00 44 5d 10 00 5a 37 00 65 37 00 65 36 00 65 37 00 3c 71 aa 03 57 68 00 00 5a	.x..dg.D]..Z7.e7.e6.e7.<q..Wh..Z
03e0	38 00 78 18 00 64 68 00 44 5d 10 00 5a 37 00 65 37 00 65 38 00 65 37 00 3c 71 cb 03 57 64 5c 00	8.x..dh.D]..Z7.e7.e8.e7.<q..Wd\.
0400	5a 39 00 64 5d 00 65 3a 00 66 01 00 64 5e 00 84 00 00 83 00 00 59 5a 3b 00 64 5f 00 65 3a 00 66	Z9.d].e:.f..d^.......YZ;.d_.e:.f
0420	01 00 64 60 00 84 00 00 83 00 00 59 5a 3c 00 64 61 00 84 00 00 5a 3d 00 65 3d 00 64 62 00 84 01	..d`.......YZ<.da....Z=.e=.db...
0440	00 5a 3e 00 64 63 00 84 00 00 5a 3f 00 64 64 00 84 00 00 5a 40 00 64 65 00 84 00 00 5a 41 00 65	.Z>.dc....Z?.dd....Z@.de....ZA.e
0460	42 00 64 66 00 6a 02 00 6f 54 00 01 64 03 00 64 04 00 6b 43 00 5a 43 00 65 44 00 65 43 00 69 45	B.df.j..oT..d..d..kC.ZC.eD.eC.iE
0480	00 83 01 00 64 0d 00 6a 04 00 6f 1e 00 01 65 3e 00 65 46 00 65 43 00 69 45 00 64 0d 00 19 83 01	....d..j..o...e>.eF.eC.iE.d.....
04a0	00 69 47 00 83 01 00 01 71 a2 04 01 65 3e 00 65 43 00 69 48 00 69 47 00 83 01 00 01 6e 01 00 01	.iG.....q...e>.eC.iH.iG.....n...
04c0	64 04 00 53 28 69 00 00 00 73 c0 03 00 00 54 6f 6b 65 6e 69 7a 61 74 69 6f 6e 20 68 65 6c 70 20	d..S(i...s....Tokenization.help.
04e0	66 6f 72 20 50 79 74 68 6f 6e 20 70 72 6f 67 72 61 6d 73 2e 0a 0a 67 65 6e 65 72 61 74 65 5f 74	for.Python.programs...generate_t
0500	6f 6b 65 6e 73 28 72 65 61 64 6c 69 6e 65 29 20 69 73 20 61 20 67 65 6e 65 72 61 74 6f 72 20 74	okens(readline).is.a.generator.t
0520	68 61 74 20 62 72 65 61 6b 73 20 61 20 73 74 72 65 61 6d 20 6f 66 0a 74 65 78 74 20 69 6e 74 6f	hat.breaks.a.stream.of.text.into
0540	20 50 79 74 68 6f 6e 20 74 6f 6b 65 6e 73 2e 20 20 49 74 20 61 63 63 65 70 74 73 20 61 20 72 65	.Python.tokens...It.accepts.a.re
0560	61 64 6c 69 6e 65 2d 6c 69 6b 65 20 6d 65 74 68 6f 64 20 77 68 69 63 68 20 69 73 20 63 61 6c 6c	adline-like.method.which.is.call
0580	65 64 0a 72 65 70 65 61 74 65 64 6c 79 20 74 6f 20 67 65 74 20 74 68 65 20 6e 65 78 74 20 6c 69	ed.repeatedly.to.get.the.next.li
05a0	6e 65 20 6f 66 20 69 6e 70 75 74 20 28 6f 72 20 22 22 20 66 6f 72 20 45 4f 46 29 2e 20 20 49 74	ne.of.input.(or."".for.EOF)...It
05c0	20 67 65 6e 65 72 61 74 65 73 0a 35 2d 74 75 70 6c 65 73 20 77 69 74 68 20 74 68 65 73 65 20 6d	.generates.5-tuples.with.these.m
05e0	65 6d 62 65 72 73 3a 0a 0a 20 20 20 20 74 68 65 20 74 6f 6b 65 6e 20 74 79 70 65 20 28 73 65 65	embers:......the.token.type.(see
0600	20 74 6f 6b 65 6e 2e 70 79 29 0a 20 20 20 20 74 68 65 20 74 6f 6b 65 6e 20 28 61 20 73 74 72 69	.token.py).....the.token.(a.stri
0620	6e 67 29 0a 20 20 20 20 74 68 65 20 73 74 61 72 74 69 6e 67 20 28 72 6f 77 2c 20 63 6f 6c 75 6d	ng).....the.starting.(row,.colum
0640	6e 29 20 69 6e 64 69 63 65 73 20 6f 66 20 74 68 65 20 74 6f 6b 65 6e 20 28 61 20 32 2d 74 75 70	n).indices.of.the.token.(a.2-tup
0660	6c 65 20 6f 66 20 69 6e 74 73 29 0a 20 20 20 20 74 68 65 20 65 6e 64 69 6e 67 20 28 72 6f 77 2c	le.of.ints).....the.ending.(row,
0680	20 63 6f 6c 75 6d 6e 29 20 69 6e 64 69 63 65 73 20 6f 66 20 74 68 65 20 74 6f 6b 65 6e 20 28 61	.column).indices.of.the.token.(a
06a0	20 32 2d 74 75 70 6c 65 20 6f 66 20 69 6e 74 73 29 0a 20 20 20 20 74 68 65 20 6f 72 69 67 69 6e	.2-tuple.of.ints).....the.origin
06c0	61 6c 20 6c 69 6e 65 20 28 73 74 72 69 6e 67 29 0a 0a 49 74 20 69 73 20 64 65 73 69 67 6e 65 64	al.line.(string)..It.is.designed
06e0	20 74 6f 20 6d 61 74 63 68 20 74 68 65 20 77 6f 72 6b 69 6e 67 20 6f 66 20 74 68 65 20 50 79 74	.to.match.the.working.of.the.Pyt
0700	68 6f 6e 20 74 6f 6b 65 6e 69 7a 65 72 20 65 78 61 63 74 6c 79 2c 20 65 78 63 65 70 74 0a 74 68	hon.tokenizer.exactly,.except.th
0720	61 74 20 69 74 20 70 72 6f 64 75 63 65 73 20 43 4f 4d 4d 45 4e 54 20 74 6f 6b 65 6e 73 20 66 6f	at.it.produces.COMMENT.tokens.fo
0740	72 20 63 6f 6d 6d 65 6e 74 73 20 61 6e 64 20 67 69 76 65 73 20 74 79 70 65 20 4f 50 20 66 6f 72	r.comments.and.gives.type.OP.for
0760	20 61 6c 6c 0a 6f 70 65 72 61 74 6f 72 73 0a 0a 4f 6c 64 65 72 20 65 6e 74 72 79 20 70 6f 69 6e	.all.operators..Older.entry.poin
0780	74 73 0a 20 20 20 20 74 6f 6b 65 6e 69 7a 65 5f 6c 6f 6f 70 28 72 65 61 64 6c 69 6e 65 2c 20 74	ts.....tokenize_loop(readline,.t
07a0	6f 6b 65 6e 65 61 74 65 72 29 0a 20 20 20 20 74 6f 6b 65 6e 69 7a 65 28 72 65 61 64 6c 69 6e 65	okeneater).....tokenize(readline
07c0	2c 20 74 6f 6b 65 6e 65 61 74 65 72 3d 70 72 69 6e 74 74 6f 6b 65 6e 29 0a 61 72 65 20 74 68 65	,.tokeneater=printtoken).are.the
07e0	20 73 61 6d 65 2c 20 65 78 63 65 70 74 20 69 6e 73 74 65 61 64 20 6f 66 20 67 65 6e 65 72 61 74	.same,.except.instead.of.generat
0800	69 6e 67 20 74 6f 6b 65 6e 73 2c 20 74 6f 6b 65 6e 65 61 74 65 72 20 69 73 20 61 20 63 61 6c 6c	ing.tokens,.tokeneater.is.a.call
0820	62 61 63 6b 0a 66 75 6e 63 74 69 6f 6e 20 74 6f 20 77 68 69 63 68 20 74 68 65 20 35 20 66 69 65	back.function.to.which.the.5.fie
0840	6c 64 73 20 64 65 73 63 72 69 62 65 64 20 61 62 6f 76 65 20 61 72 65 20 70 61 73 73 65 64 20 61	lds.described.above.are.passed.a
0860	73 20 35 20 61 72 67 75 6d 65 6e 74 73 2c 0a 65 61 63 68 20 74 69 6d 65 20 61 20 6e 65 77 20 74	s.5.arguments,.each.time.a.new.t
0880	6f 6b 65 6e 20 69 73 20 66 6f 75 6e 64 2e 73 1a 00 00 00 4b 61 2d 50 69 6e 67 20 59 65 65 20 3c	oken.is.found.s....Ka-Ping.Yee.<
08a0	70 69 6e 67 40 6c 66 77 2e 6f 72 67 3e 73 40 00 00 00 47 76 52 2c 20 45 53 52 2c 20 54 69 6d 20	ping@lfw.org>s@...GvR,.ESR,.Tim.
08c0	50 65 74 65 72 73 2c 20 54 68 6f 6d 61 73 20 57 6f 75 74 65 72 73 2c 20 46 72 65 64 20 44 72 61	Peters,.Thomas.Wouters,.Fred.Dra
08e0	6b 65 2c 20 53 6b 69 70 20 4d 6f 6e 74 61 6e 61 72 6f 69 ff ff ff ff 4e 28 01 00 00 00 74 01 00	ke,.Skip.Montanaroi....N(....t..
0900	00 00 2a 69 00 00 00 00 74 01 00 00 00 5f 74 07 00 00 00 43 4f 4d 4d 45 4e 54 74 08 00 00 00 74	..*i....t...._t....COMMENTt....t
0920	6f 6b 65 6e 69 7a 65 74 0f 00 00 00 67 65 6e 65 72 61 74 65 5f 74 6f 6b 65 6e 73 74 02 00 00 00	okenizet....generate_tokenst....
0940	4e 4c 74 0a 00 00 00 75 6e 74 6f 6b 65 6e 69 7a 65 69 01 00 00 00 69 02 00 00 00 63 00 00 00 00	NLt....untokenizei....i....c....
0960	01 00 00 00 03 00 00 00 47 00 00 00 73 15 00 00 00 64 01 00 64 02 00 69 00 00 7c 00 00 83 01 00	........G...s....d..d..i..|.....
0980	17 64 03 00 17 53 28 04 00 00 00 4e 74 01 00 00 00 28 74 01 00 00 00 7c 74 01 00 00 00 29 28 01	.d...S(....Nt....(t....|t....)(.
09a0	00 00 00 74 04 00 00 00 6a 6f 69 6e 28 01 00 00 00 74 07 00 00 00 63 68 6f 69 63 65 73 28 00 00	...t....join(....t....choices(..
09c0	00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65	..(....s..../sys/lib/python/toke
09e0	6e 69 7a 65 2e 70 79 74 05 00 00 00 67 72 6f 75 70 2c 00 00 00 73 00 00 00 00 63 00 00 00 00 01	nize.pyt....group,...s....c.....
0a00	00 00 00 02 00 00 00 47 00 00 00 73 0e 00 00 00 74 00 00 7c 00 00 8c 00 00 64 01 00 17 53 28 02	.......G...s....t..|.....d...S(.
0a20	00 00 00 4e 52 00 00 00 00 28 01 00 00 00 52 0c 00 00 00 28 01 00 00 00 52 0b 00 00 00 28 00 00	...NR....(....R....(....R....(..
0a40	00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65	..(....s..../sys/lib/python/toke
0a60	6e 69 7a 65 2e 70 79 74 03 00 00 00 61 6e 79 2d 00 00 00 73 00 00 00 00 63 00 00 00 00 01 00 00	nize.pyt....any-...s....c.......
0a80	00 02 00 00 00 47 00 00 00 73 0e 00 00 00 74 00 00 7c 00 00 8c 00 00 64 01 00 17 53 28 02 00 00	.....G...s....t..|.....d...S(...
0aa0	00 4e 74 01 00 00 00 3f 28 01 00 00 00 52 0c 00 00 00 28 01 00 00 00 52 0b 00 00 00 28 00 00 00	.Nt....?(....R....(....R....(...
0ac0	00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65 6e	.(....s..../sys/lib/python/token
0ae0	69 7a 65 2e 70 79 74 05 00 00 00 6d 61 79 62 65 2e 00 00 00 73 00 00 00 00 73 08 00 00 00 5b 20	ize.pyt....maybe....s....s....[.
0b00	5c 66 5c 74 5d 2a 73 09 00 00 00 23 5b 5e 5c 72 5c 6e 5d 2a 73 07 00 00 00 5c 5c 5c 72 3f 5c 6e	\f\t]*s....#[^\r\n]*s....\\\r?\n
0b20	73 0c 00 00 00 5b 61 2d 7a 41 2d 5a 5f 5d 5c 77 2a 73 15 00 00 00 30 5b 78 58 5d 5b 5c 64 61 2d	s....[a-zA-Z_]\w*s....0[xX][\da-
0b40	66 41 2d 46 5d 2a 5b 6c 4c 5d 3f 73 0c 00 00 00 30 5b 30 2d 37 5d 2a 5b 6c 4c 5d 3f 73 0d 00 00	fA-F]*[lL]?s....0[0-7]*[lL]?s...
0b60	00 5b 31 2d 39 5d 5c 64 2a 5b 6c 4c 5d 3f 73 0c 00 00 00 5b 65 45 5d 5b 2d 2b 5d 3f 5c 64 2b 73	.[1-9]\d*[lL]?s....[eE][-+]?\d+s
0b80	08 00 00 00 5c 64 2b 5c 2e 5c 64 2a 73 05 00 00 00 5c 2e 5c 64 2b 73 03 00 00 00 5c 64 2b 73 07	....\d+\.\d*s....\.\d+s....\d+s.
0ba0	00 00 00 5c 64 2b 5b 6a 4a 5d 73 04 00 00 00 5b 6a 4a 5d 73 17 00 00 00 5b 5e 27 5c 5c 5d 2a 28	...\d+[jJ]s....[jJ]s....[^'\\]*(
0bc0	3f 3a 5c 5c 2e 5b 5e 27 5c 5c 5d 2a 29 2a 27 73 17 00 00 00 5b 5e 22 5c 5c 5d 2a 28 3f 3a 5c 5c	?:\\.[^'\\]*)*'s....[^"\\]*(?:\\
0be0	2e 5b 5e 22 5c 5c 5d 2a 29 2a 22 73 25 00 00 00 5b 5e 27 5c 5c 5d 2a 28 3f 3a 28 3f 3a 5c 5c 2e	.[^"\\]*)*"s%...[^'\\]*(?:(?:\\.
0c00	7c 27 28 3f 21 27 27 29 29 5b 5e 27 5c 5c 5d 2a 29 2a 27 27 27 73 25 00 00 00 5b 5e 22 5c 5c 5d	|'(?!''))[^'\\]*)*'''s%...[^"\\]
0c20	2a 28 3f 3a 28 3f 3a 5c 5c 2e 7c 22 28 3f 21 22 22 29 29 5b 5e 22 5c 5c 5d 2a 29 2a 22 22 22 73	*(?:(?:\\.|"(?!""))[^"\\]*)*"""s
0c40	0d 00 00 00 5b 75 55 5d 3f 5b 72 52 5d 3f 27 27 27 73 0d 00 00 00 5b 75 55 5d 3f 5b 72 52 5d 3f	....[uU]?[rR]?'''s....[uU]?[rR]?
0c60	22 22 22 73 26 00 00 00 5b 75 55 5d 3f 5b 72 52 5d 3f 27 5b 5e 5c 6e 27 5c 5c 5d 2a 28 3f 3a 5c	"""s&...[uU]?[rR]?'[^\n'\\]*(?:\
0c80	5c 2e 5b 5e 5c 6e 27 5c 5c 5d 2a 29 2a 27 73 26 00 00 00 5b 75 55 5d 3f 5b 72 52 5d 3f 22 5b 5e	\.[^\n'\\]*)*'s&...[uU]?[rR]?"[^
0ca0	5c 6e 22 5c 5c 5d 2a 28 3f 3a 5c 5c 2e 5b 5e 5c 6e 22 5c 5c 5d 2a 29 2a 22 73 06 00 00 00 5c 2a	\n"\\]*(?:\\.[^\n"\\]*)*"s....\*
0cc0	5c 2a 3d 3f 73 04 00 00 00 3e 3e 3d 3f 73 04 00 00 00 3c 3c 3d 3f 73 02 00 00 00 3c 3e 73 02 00	\*=?s....>>=?s....<<=?s....<>s..
0ce0	00 00 21 3d 73 04 00 00 00 2f 2f 3d 3f 73 10 00 00 00 5b 2b 5c 2d 2a 2f 25 26 7c 5e 3d 3c 3e 5d	..!=s....//=?s....[+\-*/%&|^=<>]
0d00	3d 3f 74 01 00 00 00 7e 73 08 00 00 00 5b 5d 5b 28 29 7b 7d 5d 73 05 00 00 00 5c 72 3f 5c 6e 73	=?t....~s....[][(){}]s....\r?\ns
0d20	08 00 00 00 5b 3a 3b 2e 2c 60 40 5d 73 25 00 00 00 5b 75 55 5d 3f 5b 72 52 5d 3f 27 5b 5e 5c 6e	....[:;.,`@]s%...[uU]?[rR]?'[^\n
0d40	27 5c 5c 5d 2a 28 3f 3a 5c 5c 2e 5b 5e 5c 6e 27 5c 5c 5d 2a 29 2a 74 01 00 00 00 27 73 25 00 00	'\\]*(?:\\.[^\n'\\]*)*t....'s%..
0d60	00 5b 75 55 5d 3f 5b 72 52 5d 3f 22 5b 5e 5c 6e 22 5c 5c 5d 2a 28 3f 3a 5c 5c 2e 5b 5e 5c 6e 22	.[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"
0d80	5c 5c 5d 2a 29 2a 74 01 00 00 00 22 73 03 00 00 00 27 27 27 73 03 00 00 00 22 22 22 73 04 00 00	\\]*)*t...."s....'''s...."""s...
0da0	00 72 27 27 27 73 04 00 00 00 72 22 22 22 73 04 00 00 00 75 27 27 27 73 04 00 00 00 75 22 22 22	.r'''s....r"""s....u'''s....u"""
0dc0	73 05 00 00 00 75 72 27 27 27 73 05 00 00 00 75 72 22 22 22 73 04 00 00 00 52 27 27 27 73 04 00	s....ur'''s....ur"""s....R'''s..
0de0	00 00 52 22 22 22 73 04 00 00 00 55 27 27 27 73 04 00 00 00 55 22 22 22 73 05 00 00 00 75 52 27	..R"""s....U'''s....U"""s....uR'
0e00	27 27 73 05 00 00 00 75 52 22 22 22 73 05 00 00 00 55 72 27 27 27 73 05 00 00 00 55 72 22 22 22	''s....uR"""s....Ur'''s....Ur"""
0e20	73 05 00 00 00 55 52 27 27 27 73 05 00 00 00 55 52 22 22 22 74 01 00 00 00 72 74 01 00 00 00 52	s....UR'''s....UR"""t....rt....R
0e40	74 01 00 00 00 75 74 01 00 00 00 55 73 02 00 00 00 72 27 73 02 00 00 00 72 22 73 02 00 00 00 52	t....ut....Us....r's....r"s....R
0e60	27 73 02 00 00 00 52 22 73 02 00 00 00 75 27 73 02 00 00 00 75 22 73 02 00 00 00 55 27 73 02 00	's....R"s....u's....u"s....U's..
0e80	00 00 55 22 73 03 00 00 00 75 72 27 73 03 00 00 00 75 72 22 73 03 00 00 00 55 72 27 73 03 00 00	..U"s....ur's....ur"s....Ur's...
0ea0	00 55 72 22 73 03 00 00 00 75 52 27 73 03 00 00 00 75 52 22 73 03 00 00 00 55 52 27 73 03 00 00	.Ur"s....uR's....uR"s....UR's...
0ec0	00 55 52 22 69 08 00 00 00 74 0a 00 00 00 54 6f 6b 65 6e 45 72 72 6f 72 63 00 00 00 00 00 00 00	.UR"i....t....TokenErrorc.......
0ee0	00 01 00 00 00 42 00 00 00 73 08 00 00 00 65 00 00 5a 01 00 52 53 28 00 00 00 00 28 02 00 00 00	.....B...s....e..Z..RS(....(....
0f00	74 08 00 00 00 5f 5f 6e 61 6d 65 5f 5f 74 0a 00 00 00 5f 5f 6d 6f 64 75 6c 65 5f 5f 28 00 00 00	t....__name__t....__module__(...
0f20	00 28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f	.(....(....s..../sys/lib/python/
0f40	74 6f 6b 65 6e 69 7a 65 2e 70 79 52 17 00 00 00 83 00 00 00 73 00 00 00 00 74 0e 00 00 00 53 74	tokenize.pyR........s....t....St
0f60	6f 70 54 6f 6b 65 6e 69 7a 69 6e 67 63 00 00 00 00 00 00 00 00 01 00 00 00 42 00 00 00 73 08 00	opTokenizingc............B...s..
0f80	00 00 65 00 00 5a 01 00 52 53 28 00 00 00 00 28 02 00 00 00 52 18 00 00 00 52 19 00 00 00 28 00	..e..Z..RS(....(....R....R....(.
0fa0	00 00 00 28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f	...(....(....s..../sys/lib/pytho
0fc0	6e 2f 74 6f 6b 65 6e 69 7a 65 2e 70 79 52 1a 00 00 00 85 00 00 00 73 00 00 00 00 63 05 00 00 00	n/tokenize.pyR........s....c....
0fe0	09 00 00 00 08 00 00 00 43 00 00 00 73 41 00 00 00 7c 02 00 5c 02 00 7d 05 00 7d 06 00 7c 03 00	........C...sA...|..\..}..}..|..
1000	5c 02 00 7d 07 00 7d 08 00 64 01 00 7c 05 00 7c 06 00 7c 07 00 7c 08 00 74 00 00 7c 00 00 19 74	\..}..}..d..|..|..|..|..t..|...t
1020	01 00 7c 01 00 83 01 00 66 06 00 16 47 48 64 00 00 53 28 02 00 00 00 4e 73 12 00 00 00 25 64 2c	..|.....f...GHd..S(....Ns....%d,
1040	25 64 2d 25 64 2c 25 64 3a 09 25 73 09 25 73 28 02 00 00 00 74 08 00 00 00 74 6f 6b 5f 6e 61 6d	%d-%d,%d:.%s.%s(....t....tok_nam
1060	65 74 04 00 00 00 72 65 70 72 28 09 00 00 00 74 04 00 00 00 74 79 70 65 74 05 00 00 00 74 6f 6b	et....repr(....t....typet....tok
1080	65 6e 74 02 00 00 00 2e 32 74 02 00 00 00 2e 33 74 04 00 00 00 6c 69 6e 65 74 04 00 00 00 73 72	ent.....2t.....3t....linet....sr
10a0	6f 77 74 04 00 00 00 73 63 6f 6c 74 04 00 00 00 65 72 6f 77 74 04 00 00 00 65 63 6f 6c 28 00 00	owt....scolt....erowt....ecol(..
10c0	00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65	..(....s..../sys/lib/python/toke
10e0	6e 69 7a 65 2e 70 79 74 0a 00 00 00 70 72 69 6e 74 74 6f 6b 65 6e 87 00 00 00 73 04 00 00 00 18	nize.pyt....printtoken....s.....
1100	01 03 01 63 02 00 00 00 02 00 00 00 06 00 00 00 43 00 00 00 73 2b 00 00 00 79 11 00 74 00 00 7c	...c............C...s+...y..t..|
1120	00 00 7c 01 00 83 02 00 01 57 6e 13 00 04 74 01 00 6a 0a 00 6f 07 00 01 01 01 01 6e 02 00 01 58	..|......Wn...t..j..o......n...X
1140	64 01 00 53 28 02 00 00 00 73 3a 02 00 00 0a 20 20 20 20 54 68 65 20 74 6f 6b 65 6e 69 7a 65 28	d..S(....s:........The.tokenize(
1160	29 20 66 75 6e 63 74 69 6f 6e 20 61 63 63 65 70 74 73 20 74 77 6f 20 70 61 72 61 6d 65 74 65 72	).function.accepts.two.parameter
1180	73 3a 20 6f 6e 65 20 72 65 70 72 65 73 65 6e 74 69 6e 67 20 74 68 65 0a 20 20 20 20 69 6e 70 75	s:.one.representing.the.....inpu
11a0	74 20 73 74 72 65 61 6d 2c 20 61 6e 64 20 6f 6e 65 20 70 72 6f 76 69 64 69 6e 67 20 61 6e 20 6f	t.stream,.and.one.providing.an.o
11c0	75 74 70 75 74 20 6d 65 63 68 61 6e 69 73 6d 20 66 6f 72 20 74 6f 6b 65 6e 69 7a 65 28 29 2e 0a	utput.mechanism.for.tokenize()..
11e0	0a 20 20 20 20 54 68 65 20 66 69 72 73 74 20 70 61 72 61 6d 65 74 65 72 2c 20 72 65 61 64 6c 69	.....The.first.parameter,.readli
1200	6e 65 2c 20 6d 75 73 74 20 62 65 20 61 20 63 61 6c 6c 61 62 6c 65 20 6f 62 6a 65 63 74 20 77 68	ne,.must.be.a.callable.object.wh
1220	69 63 68 20 70 72 6f 76 69 64 65 73 0a 20 20 20 20 74 68 65 20 73 61 6d 65 20 69 6e 74 65 72 66	ich.provides.....the.same.interf
1240	61 63 65 20 61 73 20 74 68 65 20 72 65 61 64 6c 69 6e 65 28 29 20 6d 65 74 68 6f 64 20 6f 66 20	ace.as.the.readline().method.of.
1260	62 75 69 6c 74 2d 69 6e 20 66 69 6c 65 20 6f 62 6a 65 63 74 73 2e 0a 20 20 20 20 45 61 63 68 20	built-in.file.objects......Each.
1280	63 61 6c 6c 20 74 6f 20 74 68 65 20 66 75 6e 63 74 69 6f 6e 20 73 68 6f 75 6c 64 20 72 65 74 75	call.to.the.function.should.retu
12a0	72 6e 20 6f 6e 65 20 6c 69 6e 65 20 6f 66 20 69 6e 70 75 74 20 61 73 20 61 20 73 74 72 69 6e 67	rn.one.line.of.input.as.a.string
12c0	2e 0a 0a 20 20 20 20 54 68 65 20 73 65 63 6f 6e 64 20 70 61 72 61 6d 65 74 65 72 2c 20 74 6f 6b	.......The.second.parameter,.tok
12e0	65 6e 65 61 74 65 72 2c 20 6d 75 73 74 20 61 6c 73 6f 20 62 65 20 61 20 63 61 6c 6c 61 62 6c 65	eneater,.must.also.be.a.callable
1300	20 6f 62 6a 65 63 74 2e 20 49 74 20 69 73 0a 20 20 20 20 63 61 6c 6c 65 64 20 6f 6e 63 65 20 66	.object..It.is.....called.once.f
1320	6f 72 20 65 61 63 68 20 74 6f 6b 65 6e 2c 20 77 69 74 68 20 66 69 76 65 20 61 72 67 75 6d 65 6e	or.each.token,.with.five.argumen
1340	74 73 2c 20 63 6f 72 72 65 73 70 6f 6e 64 69 6e 67 20 74 6f 20 74 68 65 0a 20 20 20 20 74 75 70	ts,.corresponding.to.the.....tup
1360	6c 65 73 20 67 65 6e 65 72 61 74 65 64 20 62 79 20 67 65 6e 65 72 61 74 65 5f 74 6f 6b 65 6e 73	les.generated.by.generate_tokens
1380	28 29 2e 0a 20 20 20 20 4e 28 02 00 00 00 74 0d 00 00 00 74 6f 6b 65 6e 69 7a 65 5f 6c 6f 6f 70	()......N(....t....tokenize_loop
13a0	52 1a 00 00 00 28 02 00 00 00 74 08 00 00 00 72 65 61 64 6c 69 6e 65 74 0a 00 00 00 74 6f 6b 65	R....(....t....readlinet....toke
13c0	6e 65 61 74 65 72 28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79	neater(....(....s..../sys/lib/py
13e0	74 68 6f 6e 2f 74 6f 6b 65 6e 69 7a 65 2e 70 79 52 03 00 00 00 8b 00 00 00 73 08 00 00 00 00 0d	thon/tokenize.pyR........s......
1400	03 01 11 01 0e 01 63 02 00 00 00 03 00 00 00 03 00 00 00 43 00 00 00 73 25 00 00 00 78 1e 00 74	......c............C...s%...x..t
1420	00 00 7c 00 00 83 01 00 44 5d 10 00 7d 02 00 7c 01 00 7c 02 00 8c 00 00 01 71 0d 00 57 64 00 00	..|.....D]..}..|..|......q..Wd..
1440	53 28 01 00 00 00 4e 28 01 00 00 00 52 04 00 00 00 28 03 00 00 00 52 28 00 00 00 52 29 00 00 00	S(....N(....R....(....R(...R)...
1460	74 0a 00 00 00 74 6f 6b 65 6e 5f 69 6e 66 6f 28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73	t....token_info(....(....s..../s
1480	79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65 6e 69 7a 65 2e 70 79 52 27 00 00 00 9e 00	ys/lib/python/tokenize.pyR'.....
14a0	00 00 73 06 00 00 00 00 01 0d 00 06 01 63 01 00 00 00 08 00 00 00 06 00 00 00 43 00 00 00 73 f9	..s..........c............C...s.
14c0	00 00 00 74 00 00 7d 01 00 67 00 00 7d 02 00 67 00 00 7d 03 00 7c 03 00 69 01 00 7d 04 00 78 ce	...t..}..g..}..g..}..|..i..}..x.
14e0	00 7c 00 00 44 5d c6 00 7d 05 00 7c 05 00 64 01 00 20 5c 02 00 7d 06 00 7d 07 00 7c 06 00 74 02	.|..D]..}..|..d...\..}..}..|..t.
1500	00 74 03 00 66 02 00 6a 06 00 6f 0e 00 01 7c 07 00 64 02 00 37 7d 07 00 6e 01 00 01 7c 06 00 74	.t..f..j..o...|..d..7}..n...|..t
1520	04 00 6a 02 00 6f 14 00 01 7c 02 00 69 01 00 7c 07 00 83 01 00 01 71 22 00 6e 65 00 01 7c 06 00	..j..o...|..i..|......q".ne..|..
1540	74 05 00 6a 02 00 6f 11 00 01 7c 02 00 69 06 00 83 00 00 01 71 22 00 6e 47 00 01 7c 06 00 74 07	t..j..o...|..i......q".nG..|..t.
1560	00 74 08 00 74 09 00 66 03 00 6a 06 00 6f 0a 00 01 74 0a 00 7d 01 00 6e 27 00 01 7c 01 00 6f 1f	.t..t..f..j..o...t..}..n'..|..o.
1580	00 01 7c 02 00 6f 18 00 01 7c 04 00 7c 02 00 64 03 00 19 83 01 00 01 74 00 00 7d 01 00 6e 01 00	..|..o...|..|..d.......t..}..n..
15a0	01 7c 04 00 7c 07 00 83 01 00 01 71 22 00 57 64 04 00 69 0b 00 7c 03 00 83 01 00 53 28 05 00 00	.|..|......q".Wd..i..|.....S(...
15c0	00 73 f2 01 00 00 54 72 61 6e 73 66 6f 72 6d 20 74 6f 6b 65 6e 73 20 62 61 63 6b 20 69 6e 74 6f	.s....Transform.tokens.back.into
15e0	20 50 79 74 68 6f 6e 20 73 6f 75 72 63 65 20 63 6f 64 65 2e 0a 0a 20 20 20 20 45 61 63 68 20 65	.Python.source.code.......Each.e
1600	6c 65 6d 65 6e 74 20 72 65 74 75 72 6e 65 64 20 62 79 20 74 68 65 20 69 74 65 72 61 62 6c 65 20	lement.returned.by.the.iterable.
1620	6d 75 73 74 20 62 65 20 61 20 74 6f 6b 65 6e 20 73 65 71 75 65 6e 63 65 0a 20 20 20 20 77 69 74	must.be.a.token.sequence.....wit
1640	68 20 61 74 20 6c 65 61 73 74 20 74 77 6f 20 65 6c 65 6d 65 6e 74 73 2c 20 61 20 74 6f 6b 65 6e	h.at.least.two.elements,.a.token
1660	20 6e 75 6d 62 65 72 20 61 6e 64 20 74 6f 6b 65 6e 20 76 61 6c 75 65 2e 0a 0a 20 20 20 20 52 6f	.number.and.token.value.......Ro
1680	75 6e 64 2d 74 72 69 70 20 69 6e 76 61 72 69 61 6e 74 3a 0a 20 20 20 20 20 20 20 20 23 20 4f 75	und-trip.invariant:.........#.Ou
16a0	74 70 75 74 20 74 65 78 74 20 77 69 6c 6c 20 74 6f 6b 65 6e 69 7a 65 20 74 68 65 20 62 61 63 6b	tput.text.will.tokenize.the.back
16c0	20 74 6f 20 74 68 65 20 69 6e 70 75 74 0a 20 20 20 20 20 20 20 20 74 31 20 3d 20 5b 74 6f 6b 5b	.to.the.input.........t1.=.[tok[
16e0	3a 32 5d 20 66 6f 72 20 74 6f 6b 20 69 6e 20 67 65 6e 65 72 61 74 65 5f 74 6f 6b 65 6e 73 28 66	:2].for.tok.in.generate_tokens(f
1700	2e 72 65 61 64 6c 69 6e 65 29 5d 0a 20 20 20 20 20 20 20 20 6e 65 77 63 6f 64 65 20 3d 20 75 6e	.readline)].........newcode.=.un
1720	74 6f 6b 65 6e 69 7a 65 28 74 31 29 0a 20 20 20 20 20 20 20 20 72 65 61 64 6c 69 6e 65 20 3d 20	tokenize(t1).........readline.=.
1740	69 74 65 72 28 6e 65 77 63 6f 64 65 2e 73 70 6c 69 74 6c 69 6e 65 73 28 31 29 29 2e 6e 65 78 74	iter(newcode.splitlines(1)).next
1760	0a 20 20 20 20 20 20 20 20 74 32 20 3d 20 5b 74 6f 6b 5b 3a 32 5d 20 66 6f 72 20 74 6f 6b 69 6e	.........t2.=.[tok[:2].for.tokin
1780	20 67 65 6e 65 72 61 74 65 5f 74 6f 6b 65 6e 73 28 72 65 61 64 6c 69 6e 65 29 5d 0a 20 20 20 20	.generate_tokens(readline)].....
17a0	20 20 20 20 61 73 73 65 72 74 20 74 31 20 3d 3d 20 74 32 0a 20 20 20 20 69 02 00 00 00 74 01 00	....assert.t1.==.t2.....i....t..
17c0	00 00 20 69 ff ff ff ff 74 00 00 00 00 28 0c 00 00 00 74 05 00 00 00 46 61 6c 73 65 74 06 00 00	...i....t....(....t....Falset...
17e0	00 61 70 70 65 6e 64 74 04 00 00 00 4e 41 4d 45 74 06 00 00 00 4e 55 4d 42 45 52 74 06 00 00 00	.appendt....NAMEt....NUMBERt....
1800	49 4e 44 45 4e 54 74 06 00 00 00 44 45 44 45 4e 54 74 03 00 00 00 70 6f 70 74 07 00 00 00 4e 45	INDENTt....DEDENTt....popt....NE
1820	57 4c 49 4e 45 52 02 00 00 00 52 05 00 00 00 74 04 00 00 00 54 72 75 65 52 0a 00 00 00 28 08 00	WLINER....R....t....TrueR....(..
1840	00 00 74 08 00 00 00 69 74 65 72 61 62 6c 65 74 09 00 00 00 73 74 61 72 74 6c 69 6e 65 74 07 00	..t....iterablet....startlinet..
1860	00 00 69 6e 64 65 6e 74 73 74 04 00 00 00 74 6f 6b 73 74 0b 00 00 00 74 6f 6b 73 5f 61 70 70 65	..indentst....tokst....toks_appe
1880	6e 64 74 03 00 00 00 74 6f 6b 74 06 00 00 00 74 6f 6b 6e 75 6d 74 06 00 00 00 74 6f 6b 76 61 6c	ndt....tokt....toknumt....tokval
18a0	28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69 62 2f 70 79 74 68 6f 6e 2f 74	(....(....s..../sys/lib/python/t
18c0	6f 6b 65 6e 69 7a 65 2e 70 79 52 06 00 00 00 a3 00 00 00 73 2c 00 00 00 00 0f 06 01 06 01 06 01	okenize.pyR........s,...........
18e0	09 01 07 00 06 01 10 02 13 01 0e 02 0d 01 0d 01 07 01 0d 01 0a 01 07 01 16 01 0a 01 0e 01 0e 01	................................
1900	0a 01 0e 01 63 01 00 00 00 19 00 00 00 0c 00 00 00 63 00 00 00 73 d9 06 00 00 64 01 00 04 7d 01	....c............c...s....d...}.
1920	00 04 7d 02 00 7d 03 00 74 00 00 69 01 00 64 02 00 17 64 03 00 02 7d 04 00 7d 05 00 64 1d 00 5c	..}..}..t..i..d...d...}..}..d..\
1940	02 00 7d 06 00 7d 07 00 64 1c 00 7d 08 00 64 01 00 67 01 00 7d 09 00 78 40 06 79 0d 00 7c 00 00	..}..}..d..}..d..g..}..x@.y..|..
1960	83 00 00 7d 0a 00 57 6e 19 00 04 74 03 00 6a 0a 00 6f 0d 00 01 01 01 01 64 04 00 7d 0a 00 6e 02	...}..Wn...t..j..o......d..}..n.
1980	00 01 58 7c 01 00 64 05 00 17 7d 01 00 64 01 00 74 04 00 7c 0a 00 83 01 00 02 7d 0b 00 7d 0c 00	..X|..d...}..d..t..|......}..}..
19a0	7c 06 00 6f fa 00 01 7c 0a 00 70 13 00 01 74 05 00 64 06 00 7c 0d 00 66 02 00 82 02 00 6e 01 00	|..o...|..p...t..d..|..f.....n..
19c0	01 7c 0e 00 69 06 00 7c 0a 00 83 01 00 7d 0f 00 7c 0f 00 6f 4f 00 01 7c 0f 00 69 07 00 64 01 00	.|..i..|.....}..|..oO..|..i..d..
19e0	83 01 00 04 7d 0b 00 7d 10 00 74 08 00 7c 06 00 7c 0a 00 7c 10 00 20 17 7c 0d 00 7c 01 00 7c 10	....}..}..t..|..|..|....|..|..|.
1a00	00 66 02 00 7c 08 00 7c 0a 00 17 66 05 00 56 01 64 1e 00 5c 02 00 7d 06 00 7d 07 00 64 1c 00 7d	.f..|..|...f..V.d..\..}..}..d..}
1a20	08 00 71 76 03 01 7c 07 00 6f 59 00 01 7c 0a 00 64 07 00 1f 64 08 00 6a 03 00 6f 48 00 01 7c 0a	..qv..|..oY..|..d...d..j..oH..|.
1a40	00 64 09 00 1f 64 0a 00 6a 03 00 6f 37 00 01 74 09 00 7c 06 00 7c 0a 00 17 7c 0d 00 7c 01 00 74	.d...d..j..o7..t..|..|...|..|..t
1a60	04 00 7c 0a 00 83 01 00 66 02 00 7c 08 00 66 05 00 56 01 64 04 00 7d 06 00 64 1c 00 7d 08 00 71	..|.....f..|..f..V.d..}..d..}..q
1a80	40 00 71 76 03 01 7c 06 00 7c 0a 00 17 7d 06 00 7c 08 00 7c 0a 00 17 7d 08 00 71 40 00 6e f0 01	@.qv..|..|...}..|..|...}..q@.n..
1aa0	01 7c 02 00 64 01 00 6a 02 00 6f bc 01 01 7c 03 00 0c 6f b4 01 01 7c 0a 00 70 05 00 01 50 6e 01	.|..d..j..o...|...o...|..p...Pn.
1ac0	00 01 64 01 00 7d 11 00 78 7e 00 7c 0b 00 7c 0c 00 6a 00 00 6f 70 00 01 7c 0a 00 7c 0b 00 19 64	..d..}..x~.|..|..j..op..|..|...d
1ae0	0b 00 6a 02 00 6f 0e 00 01 7c 11 00 64 05 00 17 7d 11 00 6e 44 00 01 7c 0a 00 7c 0b 00 19 64 0c	..j..o...|..d...}..nD..|..|...d.
1b00	00 6a 02 00 6f 16 00 01 7c 11 00 74 0a 00 15 64 05 00 17 74 0a 00 14 7d 11 00 6e 1d 00 01 7c 0a	.j..o...|..t...d...t...}..n...|.
1b20	00 7c 0b 00 19 64 0d 00 6a 02 00 6f 0a 00 01 64 01 00 7d 11 00 6e 02 00 01 50 7c 0b 00 64 05 00	.|...d..j..o...d..}..n...P|..d..
1b40	17 7d 0b 00 71 b1 01 01 57 7c 0b 00 7c 0c 00 6a 02 00 6f 05 00 01 50 6e 01 00 01 7c 0a 00 7c 0b	.}..q...W|..|..j..o...Pn...|..|.
1b60	00 19 64 0e 00 6a 06 00 6f 45 00 01 74 0b 00 74 0c 00 66 02 00 7c 0a 00 7c 0b 00 19 64 0f 00 6a	..d..j..oE..t..t..f..|..|...d..j
1b80	02 00 19 7c 0a 00 7c 0b 00 1f 7c 01 00 7c 0b 00 66 02 00 7c 01 00 74 04 00 7c 0a 00 83 01 00 66	...|..|...|..|..f..|..t..|.....f
1ba0	02 00 7c 0a 00 66 05 00 56 01 71 40 00 6e 01 00 01 7c 11 00 7c 09 00 64 10 00 19 6a 04 00 6f 35	..|..f..V.q@.n...|..|..d...j..o5
1bc0	00 01 7c 09 00 69 0d 00 7c 11 00 83 01 00 01 74 0e 00 7c 0a 00 7c 0b 00 20 7c 01 00 64 01 00 66	..|..i..|......t..|..|...|..d..f
1be0	02 00 7c 01 00 7c 0b 00 66 02 00 7c 0a 00 66 05 00 56 01 6e 01 00 01 78 96 00 7c 11 00 7c 09 00	..|..|..f..|..f..V.n...x..|..|..
1c00	64 10 00 19 6a 00 00 6f 5a 00 01 7c 11 00 7c 09 00 6a 07 00 6f 1f 00 01 74 0f 00 64 11 00 64 12	d...j..oZ..|..|..j..o...t..d..d.
1c20	00 7c 01 00 7c 0b 00 7c 0a 00 66 04 00 83 02 00 82 01 00 6e 01 00 01 7c 09 00 64 10 00 20 7d 09	.|..|..|..f........n...|..d...}.
1c40	00 74 10 00 64 04 00 7c 01 00 7c 0b 00 66 02 00 7c 01 00 7c 0b 00 66 02 00 7c 0a 00 66 05 00 56	.t..d..|..|..f..|..|..f..|..f..V
1c60	01 71 e0 02 01 57 6e 27 00 01 7c 0a 00 70 19 00 01 74 05 00 64 13 00 7c 01 00 64 01 00 66 02 00	.q...Wn'..|..p...t..d..|..d..f..
1c80	66 02 00 82 02 00 6e 01 00 01 64 01 00 7d 03 00 78 04 03 7c 0b 00 7c 0c 00 6a 00 00 6f f6 02 01	f.....n...d..}..x..|..|..j..o...
1ca0	74 11 00 69 06 00 7c 0a 00 7c 0b 00 83 02 00 7d 12 00 7c 12 00 6f a7 02 01 7c 12 00 69 12 00 64	t..i..|..|.....}..|..o...|..i..d
1cc0	05 00 83 01 00 5c 02 00 7d 13 00 7d 10 00 7c 01 00 7c 13 00 66 02 00 7c 01 00 7c 10 00 66 02 00	.....\..}..}..|..|..f..|..|..f..
1ce0	7c 10 00 03 02 7d 14 00 7d 15 00 7d 0b 00 7c 0a 00 7c 13 00 7c 10 00 21 7c 0a 00 7c 13 00 19 02	|....}..}..}..|..|..|..!|..|....
1d00	7d 16 00 7d 17 00 7c 17 00 7c 05 00 6a 06 00 70 1a 00 01 7c 17 00 64 14 00 6a 02 00 6f 25 00 01	}..}..|..|..j..p...|..d..j..o%..
1d20	7c 16 00 64 14 00 6a 03 00 6f 18 00 01 74 13 00 7c 16 00 7c 14 00 7c 15 00 7c 0a 00 66 05 00 56	|..d..j..o...t..|..|..|..|..f..V
1d40	01 71 78 06 01 7c 17 00 64 15 00 6a 06 00 6f 2c 00 01 7c 02 00 64 01 00 6a 04 00 6f 07 00 01 74	.qx..|..d..j..o,..|..d..j..o...t
1d60	0b 00 70 04 00 01 74 14 00 7c 16 00 7c 14 00 7c 15 00 7c 0a 00 66 05 00 56 01 71 78 06 01 7c 17	..p...t..|..|..|..|..f..V.qx..|.
1d80	00 64 0f 00 6a 02 00 6f 18 00 01 74 0c 00 7c 16 00 7c 14 00 7c 15 00 7c 0a 00 66 05 00 56 01 71	.d..j..o...t..|..|..|..|..f..V.q
1da0	78 06 01 7c 16 00 74 15 00 6a 06 00 6f 7e 00 01 74 16 00 7c 16 00 19 7d 0e 00 7c 0e 00 69 06 00	x..|..t..j..o~..t..|...}..|..i..
1dc0	7c 0a 00 7c 0b 00 83 02 00 7d 0f 00 7c 0f 00 6f 3a 00 01 7c 0f 00 69 07 00 64 01 00 83 01 00 7d	|..|.....}..|..o:..|..i..d.....}
1de0	0b 00 7c 0a 00 7c 13 00 7c 0b 00 21 7d 16 00 74 08 00 7c 16 00 7c 14 00 7c 01 00 7c 0b 00 66 02	..|..|..|..!}..t..|..|..|..|..f.
1e00	00 7c 0a 00 66 05 00 56 01 71 42 06 01 7c 01 00 7c 13 00 66 02 00 7d 0d 00 7c 0a 00 7c 13 00 1f	.|..f..V.qB..|..|..f..}..|..|...
1e20	7d 06 00 7c 0a 00 7d 08 00 50 71 78 06 01 7c 17 00 74 17 00 6a 06 00 70 22 00 01 7c 16 00 64 16	}..|..}..Pqx..|..t..j..p"..|..d.
1e40	00 20 74 17 00 6a 06 00 70 11 00 01 7c 16 00 64 17 00 20 74 17 00 6a 06 00 6f 79 00 01 7c 16 00	..t..j..p...|..d...t..j..oy..|..
1e60	64 10 00 19 64 18 00 6a 02 00 6f 50 00 01 7c 01 00 7c 13 00 66 02 00 7d 0d 00 74 16 00 7c 17 00	d...d..j..oP..|..|..f..}..t..|..
1e80	19 70 1b 00 01 74 16 00 7c 16 00 64 05 00 19 19 70 0c 00 01 74 16 00 7c 16 00 64 16 00 19 19 7d	.p...t..|..d....p...t..|..d....}
1ea0	0e 00 7c 0a 00 7c 13 00 1f 64 05 00 02 7d 06 00 7d 07 00 7c 0a 00 7d 08 00 50 71 42 06 01 74 08	..|..|...d...}..}..|..}..PqB..t.
1ec0	00 7c 16 00 7c 14 00 7c 15 00 7c 0a 00 66 05 00 56 01 71 78 06 01 7c 17 00 7c 04 00 6a 06 00 6f	.|..|..|..|..f..V.qx..|..|..j..o
1ee0	18 00 01 74 18 00 7c 16 00 7c 14 00 7c 15 00 7c 0a 00 66 05 00 56 01 71 78 06 01 7c 17 00 64 19	...t..|..|..|..|..f..V.qx..|..d.
1f00	00 6a 02 00 6f 0a 00 01 64 05 00 7d 03 00 71 78 06 01 7c 17 00 64 1a 00 6a 06 00 6f 0e 00 01 7c	.j..o...d..}..qx..|..d..j..o...|
1f20	02 00 64 05 00 17 7d 02 00 6e 1c 00 01 7c 17 00 64 1b 00 6a 06 00 6f 0e 00 01 7c 02 00 64 05 00	..d...}..n...|..d..j..o...|..d..
1f40	18 7d 02 00 6e 01 00 01 74 19 00 7c 16 00 7c 14 00 7c 15 00 7c 0a 00 66 05 00 56 01 71 79 03 01	.}..n...t..|..|..|..|..f..V.qy..
1f60	74 09 00 7c 0a 00 7c 0b 00 19 7c 01 00 7c 0b 00 66 02 00 7c 01 00 7c 0b 00 64 05 00 17 66 02 00	t..|..|...|..|..f..|..|..d...f..
1f80	7c 0a 00 66 05 00 56 01 7c 0b 00 64 05 00 17 7d 0b 00 71 79 03 01 57 71 40 00 78 32 00 7c 09 00	|..f..V.|..d...}..qy..Wq@.x2.|..
1fa0	64 05 00 1f 44 5d 26 00 7d 18 00 74 10 00 64 04 00 7c 01 00 64 01 00 66 02 00 7c 01 00 64 01 00	d...D]&.}..t..d..|..d..f..|..d..
1fc0	66 02 00 64 04 00 66 05 00 56 01 71 8b 06 57 74 1a 00 64 04 00 7c 01 00 64 01 00 66 02 00 7c 01	f..d..f..V.q..Wt..d..|..d..f..|.
1fe0	00 64 01 00 66 02 00 64 04 00 66 05 00 56 01 64 1c 00 53 28 1f 00 00 00 73 53 03 00 00 0a 20 20	.d..f..d..f..V.d..S(....sS......
2000	20 20 54 68 65 20 67 65 6e 65 72 61 74 65 5f 74 6f 6b 65 6e 73 28 29 20 67 65 6e 65 72 61 74 6f	..The.generate_tokens().generato
2020	72 20 72 65 71 75 69 72 65 73 20 6f 6e 65 20 61 72 67 6d 65 6e 74 2c 20 72 65 61 64 6c 69 6e 65	r.requires.one.argment,.readline
2040	2c 20 77 68 69 63 68 0a 20 20 20 20 6d 75 73 74 20 62 65 20 61 20 63 61 6c 6c 61 62 6c 65 20 6f	,.which.....must.be.a.callable.o
2060	62 6a 65 63 74 20 77 68 69 63 68 20 70 72 6f 76 69 64 65 73 20 74 68 65 20 73 61 6d 65 20 69 6e	bject.which.provides.the.same.in
2080	74 65 72 66 61 63 65 20 61 73 20 74 68 65 0a 20 20 20 20 72 65 61 64 6c 69 6e 65 28 29 20 6d 65	terface.as.the.....readline().me
20a0	74 68 6f 64 20 6f 66 20 62 75 69 6c 74 2d 69 6e 20 66 69 6c 65 20 6f 62 6a 65 63 74 73 2e 20 45	thod.of.built-in.file.objects..E
20c0	61 63 68 20 63 61 6c 6c 20 74 6f 20 74 68 65 20 66 75 6e 63 74 69 6f 6e 0a 20 20 20 20 73 68 6f	ach.call.to.the.function.....sho
20e0	75 6c 64 20 72 65 74 75 72 6e 20 6f 6e 65 20 6c 69 6e 65 20 6f 66 20 69 6e 70 75 74 20 61 73 20	uld.return.one.line.of.input.as.
2100	61 20 73 74 72 69 6e 67 2e 20 20 41 6c 74 65 72 6e 61 74 65 6c 79 2c 20 72 65 61 64 6c 69 6e 65	a.string...Alternately,.readline
2120	0a 20 20 20 20 63 61 6e 20 62 65 20 61 20 63 61 6c 6c 61 62 6c 65 20 66 75 6e 63 74 69 6f 6e 20	.....can.be.a.callable.function.
2140	74 65 72 6d 69 6e 61 74 69 6e 67 20 77 69 74 68 20 53 74 6f 70 49 74 65 72 61 74 69 6f 6e 3a 0a	terminating.with.StopIteration:.
2160	20 20 20 20 20 20 20 20 72 65 61 64 6c 69 6e 65 20 3d 20 6f 70 65 6e 28 6d 79 66 69 6c 65 29 2e	........readline.=.open(myfile).
2180	6e 65 78 74 20 20 20 20 23 20 45 78 61 6d 70 6c 65 20 6f 66 20 61 6c 74 65 72 6e 61 74 65 20 72	next....#.Example.of.alternate.r
21a0	65 61 64 6c 69 6e 65 0a 0a 20 20 20 20 54 68 65 20 67 65 6e 65 72 61 74 6f 72 20 70 72 6f 64 75	eadline......The.generator.produ
21c0	63 65 73 20 35 2d 74 75 70 6c 65 73 20 77 69 74 68 20 74 68 65 73 65 20 6d 65 6d 62 65 72 73 3a	ces.5-tuples.with.these.members:
21e0	20 74 68 65 20 74 6f 6b 65 6e 20 74 79 70 65 3b 20 74 68 65 0a 20 20 20 20 74 6f 6b 65 6e 20 73	.the.token.type;.the.....token.s
2200	74 72 69 6e 67 3b 20 61 20 32 2d 74 75 70 6c 65 20 28 73 72 6f 77 2c 20 73 63 6f 6c 29 20 6f 66	tring;.a.2-tuple.(srow,.scol).of
2220	20 69 6e 74 73 20 73 70 65 63 69 66 79 69 6e 67 20 74 68 65 20 72 6f 77 20 61 6e 64 0a 20 20 20	.ints.specifying.the.row.and....
2240	20 63 6f 6c 75 6d 6e 20 77 68 65 72 65 20 74 68 65 20 74 6f 6b 65 6e 20 62 65 67 69 6e 73 20 69	.column.where.the.token.begins.i
2260	6e 20 74 68 65 20 73 6f 75 72 63 65 3b 20 61 20 32 2d 74 75 70 6c 65 20 28 65 72 6f 77 2c 20 65	n.the.source;.a.2-tuple.(erow,.e
2280	63 6f 6c 29 20 6f 66 0a 20 20 20 20 69 6e 74 73 20 73 70 65 63 69 66 79 69 6e 67 20 74 68 65 20	col).of.....ints.specifying.the.
22a0	72 6f 77 20 61 6e 64 20 63 6f 6c 75 6d 6e 20 77 68 65 72 65 20 74 68 65 20 74 6f 6b 65 6e 20 65	row.and.column.where.the.token.e
22c0	6e 64 73 20 69 6e 20 74 68 65 20 73 6f 75 72 63 65 3b 0a 20 20 20 20 61 6e 64 20 74 68 65 20 6c	nds.in.the.source;.....and.the.l
22e0	69 6e 65 20 6f 6e 20 77 68 69 63 68 20 74 68 65 20 74 6f 6b 65 6e 20 77 61 73 20 66 6f 75 6e 64	ine.on.which.the.token.was.found
2300	2e 20 54 68 65 20 6c 69 6e 65 20 70 61 73 73 65 64 20 69 73 20 74 68 65 0a 20 20 20 20 6c 6f 67	..The.line.passed.is.the.....log
2320	69 63 61 6c 20 6c 69 6e 65 3b 20 63 6f 6e 74 69 6e 75 61 74 69 6f 6e 20 6c 69 6e 65 73 20 61 72	ical.line;.continuation.lines.ar
2340	65 20 69 6e 63 6c 75 64 65 64 2e 0a 20 20 20 20 69 00 00 00 00 52 01 00 00 00 74 0a 00 00 00 30	e.included......i....R....t....0
2360	31 32 33 34 35 36 37 38 39 52 2c 00 00 00 69 01 00 00 00 73 18 00 00 00 45 4f 46 20 69 6e 20 6d	123456789R,...i....s....EOF.in.m
2380	75 6c 74 69 2d 6c 69 6e 65 20 73 74 72 69 6e 67 69 fe ff ff ff 73 02 00 00 00 5c 0a 69 fd ff ff	ulti-line.stringi....s....\.i...
23a0	ff 73 03 00 00 00 5c 0d 0a 52 2b 00 00 00 73 01 00 00 00 09 73 01 00 00 00 0c 73 03 00 00 00 23	.s....\..R+...s.....s.....s....#
23c0	0d 0a 74 01 00 00 00 23 69 ff ff ff ff 73 33 00 00 00 75 6e 69 6e 64 65 6e 74 20 64 6f 65 73 20	..t....#i....s3...unindent.does.
23e0	6e 6f 74 20 6d 61 74 63 68 20 61 6e 79 20 6f 75 74 65 72 20 69 6e 64 65 6e 74 61 74 69 6f 6e 20	not.match.any.outer.indentation.
2400	6c 65 76 65 6c 73 0a 00 00 00 3c 74 6f 6b 65 6e 69 7a 65 3e 73 1b 00 00 00 45 4f 46 20 69 6e 20	levels....<tokenize>s....EOF.in.
2420	6d 75 6c 74 69 2d 6c 69 6e 65 20 73 74 61 74 65 6d 65 6e 74 74 01 00 00 00 2e 73 02 00 00 00 0d	multi-line.statementt.....s.....
2440	0a 69 02 00 00 00 69 03 00 00 00 73 01 00 00 00 0a 73 01 00 00 00 5c 73 03 00 00 00 28 5b 7b 73	.i....i....s.....s....\s....([{s
2460	03 00 00 00 29 5d 7d 4e 28 02 00 00 00 52 2c 00 00 00 69 00 00 00 00 28 02 00 00 00 52 2c 00 00	....)]}N(....R,...i....(....R,..
2480	00 69 00 00 00 00 28 1b 00 00 00 74 06 00 00 00 73 74 72 69 6e 67 74 0d 00 00 00 61 73 63 69 69	.i....(....t....stringt....ascii
24a0	5f 6c 65 74 74 65 72 73 74 04 00 00 00 4e 6f 6e 65 74 0d 00 00 00 53 74 6f 70 49 74 65 72 61 74	_letterst....Nonet....StopIterat
24c0	69 6f 6e 74 03 00 00 00 6c 65 6e 52 17 00 00 00 74 05 00 00 00 6d 61 74 63 68 74 03 00 00 00 65	iont....lenR....t....matcht....e
24e0	6e 64 74 06 00 00 00 53 54 52 49 4e 47 74 0a 00 00 00 45 52 52 4f 52 54 4f 4b 45 4e 74 07 00 00	ndt....STRINGt....ERRORTOKENt...
2500	00 74 61 62 73 69 7a 65 52 05 00 00 00 52 02 00 00 00 52 2e 00 00 00 52 31 00 00 00 74 10 00 00	.tabsizeR....R....R....R1...t...
2520	00 49 6e 64 65 6e 74 61 74 69 6f 6e 45 72 72 6f 72 52 32 00 00 00 74 0a 00 00 00 70 73 65 75 64	.IndentationErrorR2...t....pseud
2540	6f 70 72 6f 67 74 04 00 00 00 73 70 61 6e 52 30 00 00 00 52 34 00 00 00 74 0d 00 00 00 74 72 69	oprogt....spanR0...R4...t....tri
2560	70 6c 65 5f 71 75 6f 74 65 64 74 08 00 00 00 65 6e 64 70 72 6f 67 73 74 0d 00 00 00 73 69 6e 67	ple_quotedt....endprogst....sing
2580	6c 65 5f 71 75 6f 74 65 64 52 2f 00 00 00 74 02 00 00 00 4f 50 74 09 00 00 00 45 4e 44 4d 41 52	le_quotedR/...t....OPt....ENDMAR
25a0	4b 45 52 28 19 00 00 00 52 28 00 00 00 74 04 00 00 00 6c 6e 75 6d 74 08 00 00 00 70 61 72 65 6e	KER(....R(...t....lnumt....paren
25c0	6c 65 76 74 09 00 00 00 63 6f 6e 74 69 6e 75 65 64 74 09 00 00 00 6e 61 6d 65 63 68 61 72 73 74	levt....continuedt....namecharst
25e0	08 00 00 00 6e 75 6d 63 68 61 72 73 74 07 00 00 00 63 6f 6e 74 73 74 72 74 08 00 00 00 6e 65 65	....numcharst....contstrt....nee
2600	64 63 6f 6e 74 74 08 00 00 00 63 6f 6e 74 6c 69 6e 65 52 38 00 00 00 52 21 00 00 00 74 03 00 00	dcontt....contlineR8...R!...t...
2620	00 70 6f 73 74 03 00 00 00 6d 61 78 74 08 00 00 00 73 74 72 73 74 61 72 74 74 07 00 00 00 65 6e	.post....maxt....strstartt....en
2640	64 70 72 6f 67 74 08 00 00 00 65 6e 64 6d 61 74 63 68 52 47 00 00 00 74 06 00 00 00 63 6f 6c 75	dprogt....endmatchRG...t....colu
2660	6d 6e 74 0b 00 00 00 70 73 65 75 64 6f 6d 61 74 63 68 74 05 00 00 00 73 74 61 72 74 74 04 00 00	mnt....pseudomatcht....startt...
2680	00 73 70 6f 73 74 04 00 00 00 65 70 6f 73 52 1e 00 00 00 74 07 00 00 00 69 6e 69 74 69 61 6c 74	.spost....eposR....t....initialt
26a0	06 00 00 00 69 6e 64 65 6e 74 28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f 6c 69	....indent(....(....s..../sys/li
26c0	62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65 6e 69 7a 65 2e 70 79 52 04 00 00 00 cb 00 00 00 73 d8 00	b/python/tokenize.pyR........s..
26e0	00 00 00 10 0e 01 14 01 0c 01 06 01 09 02 03 01 03 01 0d 01 0e 01 0b 01 0a 01 13 02 07 01 07 01	................................
2700	13 01 0f 01 07 01 13 01 0e 01 18 01 0c 01 0a 01 29 01 0a 01 1a 01 06 01 06 01 07 02 0a 01 0a 01	................)...............
2720	07 02 15 01 0c 01 06 01 10 01 1f 01 27 01 1b 01 01 01 0f 01 12 02 11 01 1e 01 20 01 07 02 11 01	............'...................
2740	0d 01 28 01 14 01 0d 01 03 01 03 01 19 01 0a 01 29 03 07 01 19 01 06 02 10 01 12 01 07 01 15 01	..(.............)...............
2760	20 01 18 02 0d 01 1a 01 18 01 0d 01 17 01 15 01 0d 01 18 01 0d 01 0a 01 12 01 07 01 0f 01 0d 01	................................
2780	1e 02 0c 01 0a 01 06 01 05 01 0d 01 11 01 11 01 11 01 0c 01 1a 01 0e 01 11 01 06 01 05 02 18 01	................................
27a0	0d 01 18 01 0d 01 0a 02 1b 01 1b 01 18 02 0a 01 1e 01 12 02 0b 00 06 01 24 01 74 08 00 00 00 5f	........................$.t...._
27c0	5f 6d 61 69 6e 5f 5f 28 12 00 00 00 73 03 00 00 00 27 27 27 73 03 00 00 00 22 22 22 73 04 00 00	_main__(....s....'''s...."""s...
27e0	00 72 27 27 27 73 04 00 00 00 72 22 22 22 73 04 00 00 00 52 27 27 27 73 04 00 00 00 52 22 22 22	.r'''s....r"""s....R'''s....R"""
2800	73 04 00 00 00 75 27 27 27 73 04 00 00 00 75 22 22 22 73 04 00 00 00 55 27 27 27 73 04 00 00 00	s....u'''s....u"""s....U'''s....
2820	55 22 22 22 73 05 00 00 00 75 72 27 27 27 73 05 00 00 00 75 72 22 22 22 73 05 00 00 00 55 72 27	U"""s....ur'''s....ur"""s....Ur'
2840	27 27 73 05 00 00 00 55 72 22 22 22 73 05 00 00 00 75 52 27 27 27 73 05 00 00 00 75 52 22 22 22	''s....Ur"""s....uR'''s....uR"""
2860	73 05 00 00 00 55 52 27 27 27 73 05 00 00 00 55 52 22 22 22 28 12 00 00 00 52 11 00 00 00 52 12	s....UR'''s....UR"""(....R....R.
2880	00 00 00 73 02 00 00 00 72 27 73 02 00 00 00 72 22 73 02 00 00 00 52 27 73 02 00 00 00 52 22 73	...s....r's....r"s....R's....R"s
28a0	02 00 00 00 75 27 73 02 00 00 00 75 22 73 02 00 00 00 55 27 73 02 00 00 00 55 22 73 03 00 00 00	....u's....u"s....U's....U"s....
28c0	75 72 27 73 03 00 00 00 75 72 22 73 03 00 00 00 55 72 27 73 03 00 00 00 55 72 22 73 03 00 00 00	ur's....ur"s....Ur's....Ur"s....
28e0	75 52 27 73 03 00 00 00 75 52 22 73 03 00 00 00 55 52 27 73 03 00 00 00 55 52 22 28 49 00 00 00	uR's....uR"s....UR's....UR"(I...
2900	74 07 00 00 00 5f 5f 64 6f 63 5f 5f 74 0a 00 00 00 5f 5f 61 75 74 68 6f 72 5f 5f 74 0b 00 00 00	t....__doc__t....__author__t....
2920	5f 5f 63 72 65 64 69 74 73 5f 5f 52 41 00 00 00 74 02 00 00 00 72 65 52 1e 00 00 00 74 04 00 00	__credits__RA...t....reR....t...
2940	00 5f 5b 31 5d 74 03 00 00 00 64 69 72 74 01 00 00 00 78 74 07 00 00 00 5f 5f 61 6c 6c 5f 5f 74	._[1]t....dirt....xt....__all__t
2960	08 00 00 00 4e 5f 54 4f 4b 45 4e 53 52 02 00 00 00 52 1b 00 00 00 52 05 00 00 00 52 0c 00 00 00	....N_TOKENSR....R....R....R....
2980	52 0d 00 00 00 52 0f 00 00 00 74 0a 00 00 00 57 68 69 74 65 73 70 61 63 65 74 07 00 00 00 43 6f	R....R....t....Whitespacet....Co
29a0	6d 6d 65 6e 74 74 06 00 00 00 49 67 6e 6f 72 65 74 04 00 00 00 4e 61 6d 65 74 09 00 00 00 48 65	mmentt....Ignoret....Namet....He
29c0	78 6e 75 6d 62 65 72 74 09 00 00 00 4f 63 74 6e 75 6d 62 65 72 74 09 00 00 00 44 65 63 6e 75 6d	xnumbert....Octnumbert....Decnum
29e0	62 65 72 74 09 00 00 00 49 6e 74 6e 75 6d 62 65 72 74 08 00 00 00 45 78 70 6f 6e 65 6e 74 74 0a	bert....Intnumbert....Exponentt.
2a00	00 00 00 50 6f 69 6e 74 66 6c 6f 61 74 74 08 00 00 00 45 78 70 66 6c 6f 61 74 74 0b 00 00 00 46	...Pointfloatt....Expfloatt....F
2a20	6c 6f 61 74 6e 75 6d 62 65 72 74 0a 00 00 00 49 6d 61 67 6e 75 6d 62 65 72 74 06 00 00 00 4e 75	loatnumbert....Imagnumbert....Nu
2a40	6d 62 65 72 74 06 00 00 00 53 69 6e 67 6c 65 74 06 00 00 00 44 6f 75 62 6c 65 74 07 00 00 00 53	mbert....Singlet....Doublet....S
2a60	69 6e 67 6c 65 33 74 07 00 00 00 44 6f 75 62 6c 65 33 74 06 00 00 00 54 72 69 70 6c 65 74 06 00	ingle3t....Double3t....Triplet..
2a80	00 00 53 74 72 69 6e 67 74 08 00 00 00 4f 70 65 72 61 74 6f 72 74 07 00 00 00 42 72 61 63 6b 65	..Stringt....Operatort....Bracke
2aa0	74 74 07 00 00 00 53 70 65 63 69 61 6c 74 05 00 00 00 46 75 6e 6e 79 74 0a 00 00 00 50 6c 61 69	tt....Specialt....Funnyt....Plai
2ac0	6e 54 6f 6b 65 6e 74 05 00 00 00 54 6f 6b 65 6e 74 07 00 00 00 43 6f 6e 74 53 74 72 74 0c 00 00	nTokent....Tokent....ContStrt...
2ae0	00 50 73 65 75 64 6f 45 78 74 72 61 73 74 0b 00 00 00 50 73 65 75 64 6f 54 6f 6b 65 6e 74 03 00	.PseudoExtrast....PseudoTokent..
2b00	00 00 6d 61 70 74 07 00 00 00 63 6f 6d 70 69 6c 65 74 09 00 00 00 74 6f 6b 65 6e 70 72 6f 67 52	..mapt....compilet....tokenprogR
2b20	4c 00 00 00 74 0b 00 00 00 73 69 6e 67 6c 65 33 70 72 6f 67 74 0b 00 00 00 64 6f 75 62 6c 65 33	L...t....single3progt....double3
2b40	70 72 6f 67 52 43 00 00 00 52 4f 00 00 00 52 4e 00 00 00 74 01 00 00 00 74 52 50 00 00 00 52 4a	progRC...RO...RN...t....tRP...RJ
2b60	00 00 00 74 09 00 00 00 45 78 63 65 70 74 69 6f 6e 52 17 00 00 00 52 1a 00 00 00 52 26 00 00 00	...t....ExceptionR....R....R&...
2b80	52 03 00 00 00 52 27 00 00 00 52 06 00 00 00 52 04 00 00 00 52 18 00 00 00 74 03 00 00 00 73 79	R....R'...R....R....R....t....sy
2ba0	73 52 45 00 00 00 74 04 00 00 00 61 72 67 76 74 04 00 00 00 6f 70 65 6e 52 28 00 00 00 74 05 00	sRE...t....argvt....openR(...t..
2bc0	00 00 73 74 64 69 6e 28 00 00 00 00 28 00 00 00 00 28 00 00 00 00 73 1b 00 00 00 2f 73 79 73 2f	..stdin(....(....(....s..../sys/
2be0	6c 69 62 2f 70 79 74 68 6f 6e 2f 74 6f 6b 65 6e 69 7a 65 2e 70 79 73 08 00 00 00 3c 6d 6f 64 75	lib/python/tokenize.pys....<modu
2c00	6c 65 3e 17 00 00 00 73 bc 00 00 00 06 02 06 02 06 02 18 01 0a 02 0c 01 3f 01 10 01 03 01 03 02	le>....s................?.......
2c20	06 01 0a 01 0a 01 0a 01 0a 02 09 01 09 01 09 02 06 01 06 01 1e 01 06 02 06 01 06 01 06 01 12 01	................................
2c40	06 01 19 01 0a 01 0f 01 13 01 12 03 06 02 06 02 06 02 06 01 0f 02 06 01 09 05 12 01 03 01 03 01	................................
2c60	09 02 06 01 0f 01 12 02 15 01 0a 03 06 01 0d 01 03 01 13 01 12 01 1c 02 03 01 27 01 28 01 12 01	..........................'.(...
2c80	12 01 12 01 12 01 12 01 12 01 12 01 12 01 12 01 26 02 06 01 03 01 00 01 00 01 00 01 04 00 06 01	................&...............
2ca0	0e 01 06 01 03 01 00 01 00 01 00 01 04 00 06 01 0e 02 06 02 16 02 16 02 09 04 0c 13 09 05 09 28	...............................(
2cc0	09 8b 0d 01 0c 01 34 01	......4.