Update test.py to match changed pyasn1gen.
[asn2quickder] / asn1ate / pyasn1gen.py
1 # Copyright (c) 2013-2015, Schneider Electric Buildings AB
2 # All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are met:
6 #     * Redistributions of source code must retain the above copyright
7 #       notice, this list of conditions and the following disclaimer.
8 #     * Redistributions in binary form must reproduce the above copyright
9 #       notice, this list of conditions and the following disclaimer in the
10 #       documentation and/or other materials provided with the distribution.
11 #     * Neither the name of Schneider Electric Buildings AB nor the
12 #       names of contributors may be used to endorse or promote products
13 #       derived from this software without specific prior written permission.
14 #
15 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
19 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 from __future__ import print_function  # Python 2 compatibility
27
28 import sys
29 import argparse
30 import keyword
31 from asn1ate import parser
32 from asn1ate.support import pygen
33 from asn1ate.sema import *
34
35
36 class Pyasn1Backend(object):
37     """ Backend to generate pyasn1 declarations from semantic tree.
38
39     Pyasn1 represents type assignments as class derivation, e.g.
40
41         # Foo ::= INTEGER
42         class Foo(univ.Integer):
43             pass
44
45     For constructed types, the component types are instantiated inline, e.g.
46
47         # Seq ::= SEQUENCE {
48         #     foo INTEGER
49         # }
50         class Seq(univ.Sequence):
51              componentType = namedtype.NamedTypes(
52                 namedtype.NamedType('foo', univ.Integer())
53              )
54
55     (univ.Integer is not a base class here, but a value.)
56
57     To cope with circular dependencies, we define types in two passes so we'll
58     generate the above as:
59
60         class Seq(univ.Sequence):
61             pass
62
63         Seq.componentType = namedtype.NamedTypes(
64             namedtype.NamedType('foo', univ.Integer())
65         )
66
67     This is nice, because we separate the introduction of a name (``Seq``) from
68     the details of what it contains, so we can build recursive definitions
69     without getting into trouble with Python's name lookup.
70
71     We call the empty class a *declaration*, and the population of its members
72     *definition*. The instantiation of univ.Integer is called an
73     *inline definition*.
74
75     The translation from ASN.1 constructs to Pyasn1 come in different flavors,
76     depending on whether they're declarations, definitions or inline
77     definitions.
78
79     Only type and value assignments generate declarations. For type assignments
80     we generate a definition once all dependent declarations are created. If the
81     type assignment involves a constructed type, it is filled with inline
82     definitions.
83     """
84     def __init__(self, sema_module, out_stream, referenced_modules):
85         self.sema_module = sema_module
86         self.referenced_modules = referenced_modules
87         self.writer = pygen.PythonWriter(out_stream)
88
89         self.decl_generators = {
90             TypeAssignment: self.decl_type_assignment,
91             ValueAssignment: self.decl_value_assignment
92         }
93
94         self.defn_generators = {
95             ChoiceType: self.defn_constructed_type,
96             SequenceType: self.defn_constructed_type,
97             SetType: self.defn_constructed_type,
98             SequenceOfType: self.defn_collection_type,
99             SetOfType: self.defn_collection_type,
100             TaggedType: self.defn_tagged_type,
101             SelectionType: self.defn_selection_type,
102             SimpleType: self.defn_simple_type,
103             DefinedType: self.defn_defined_type,
104             ValueListType: self.defn_value_list_type,
105             BitStringType: self.defn_bitstring_type,
106         }
107
108         self.inline_generators = {
109             TaggedType: self.inline_tagged_type,
110             SelectionType: self.inline_selection_type,
111             SimpleType: self.inline_simple_type,
112             DefinedType: self.inline_defined_type,
113             ComponentType: self.inline_component_type,
114             NamedType: self.inline_named_type,
115             SequenceOfType: self.inline_sequenceof_type,
116             SetOfType: self.inline_setof_type,
117             ValueListType: self.inline_value_list_type,
118             ChoiceType: self.inline_constructed_type,
119             SequenceType: self.inline_constructed_type,
120             SetType: self.inline_constructed_type,
121             BitStringType: self.inline_bitstring_type,
122         }
123
124     def generate_code(self):
125         self.writer.write_line('from pyasn1.type import univ, char, namedtype, namedval, tag, constraint, useful')
126         for module in self.referenced_modules:
127             if not module is self.sema_module:
128                 self.writer.write_line('import ' + _sanitize_module(module.name))
129         self.writer.write_blanks(2)
130
131         # Generate _OID if sema_module contains any object identifier values.
132         oids = [n for n in self.sema_module.descendants() if isinstance(n, ObjectIdentifierValue)]
133         if oids:
134             self.writer.write_block(self.generate_OID())
135             self.writer.write_blanks(2)
136
137         assignment_components = dependency_sort(self.sema_module.assignments)
138         for component in assignment_components:
139             for assignment in component:
140                 self.writer.write_block(self.generate_decl(assignment))
141                 self.writer.write_blanks(2)
142
143             for assignment in component:
144                 details = self.generate_definition(assignment)
145                 if details:
146                     self.writer.write_block(details)
147                     self.writer.write_blanks(2)
148
149     def generate_definition(self, assignment):
150         assert isinstance(assignment, (ValueAssignment, TypeAssignment))
151
152         if isinstance(assignment, ValueAssignment):
153             return None  # Nothing to do here.
154
155         assigned_type, type_decl = assignment.type_name, assignment.type_decl
156         assigned_type = _translate_type(assigned_type)
157         return self.generate_defn(assigned_type, type_decl)
158
159     def generate_decl(self, t):
160         generator = self.decl_generators[type(t)]
161         return generator(t)
162
163     def generate_expr(self, t):
164         generator = self.inline_generators[type(t)]
165         return generator(t)
166
167     def generate_defn(self, class_name, t):
168         generator = self.defn_generators[type(t)]
169         return generator(class_name, t)
170
171     def decl_type_assignment(self, assignment):
172         fragment = self.writer.get_fragment()
173
174         assigned_type, type_decl = assignment.type_name, assignment.type_decl
175
176         if isinstance(type_decl, SelectionType):
177             type_decl = self.sema_module.resolve_selection_type(type_decl)
178
179         assigned_type = _translate_type(assigned_type)
180         base_type = _translate_type(type_decl.type_name)
181         fragment.write_line('class %s(%s):' % (assigned_type, base_type))
182         fragment.push_indent()
183         fragment.write_line('pass')
184         fragment.pop_indent()
185
186         return str(fragment)
187
188     def decl_value_assignment(self, assignment):
189         assigned_value, type_decl, value = assignment.value_name, assignment.type_decl, assignment.value
190         assigned_value = _sanitize_identifier(assigned_value)
191         construct_expr = self.build_value_construct_expr(type_decl, value)
192         return '%s = %s' % (assigned_value, construct_expr)
193
194     def defn_simple_type(self, class_name, t):
195         if t.constraint:
196             return '%s.subtypeSpec = %s' % (class_name, self.build_constraint_expr(t.constraint))
197
198         return None
199
200     def defn_defined_type(self, class_name, t):
201         return None
202
203     def defn_constructed_type(self, class_name, t):
204         fragment = self.writer.get_fragment()
205
206         fragment.write_line('%s.componentType = namedtype.NamedTypes(' % class_name)
207         fragment.push_indent()
208         fragment.write_block(self.inline_component_types(t.components))
209         fragment.pop_indent()
210         fragment.write_line(')')
211
212         return str(fragment)
213
214     def defn_tagged_type(self, class_name, t):
215         fragment = self.writer.get_fragment()
216
217         implicity = self.sema_module.resolve_tag_implicity(t.implicity, t.type_decl)
218         if implicity == TagImplicity.IMPLICIT:
219             tag_implicity = 'tagImplicitly'
220         elif implicity == TagImplicity.EXPLICIT:
221             tag_implicity = 'tagExplicitly'
222         else:
223             assert False, "Unexpected implicity: %s" % implicity
224
225         base_type = _translate_type(t.type_decl.type_name)
226
227         fragment.write_line('%s.tagSet = %s.tagSet.%s(%s)' % (class_name, base_type, tag_implicity, self.build_tag_expr(t)))
228         nested_dfn = self.generate_defn(class_name, t.type_decl)
229         if nested_dfn:
230             fragment.write_line(nested_dfn)
231
232         return str(fragment)
233
234     def defn_selection_type(self, class_name, t):
235         return None
236
237     def defn_value_list_type(self, class_name, t):
238         fragment = self.writer.get_fragment()
239
240         if t.named_values:
241             fragment.write_line('%s.namedValues = namedval.NamedValues(' % class_name)
242             fragment.push_indent()
243
244             named_values = ['(\'%s\', %s)' % (v.identifier, v.value) for v in t.named_values if not isinstance(v, ExtensionMarker)]
245             fragment.write_enumeration(named_values)
246
247             fragment.pop_indent()
248             fragment.write_line(')')
249
250         if t.constraint:
251             fragment.write_line('%s.subtypeSpec=%s' % (class_name, self.build_constraint_expr(t.constraint)))
252
253         return str(fragment)
254
255     def inline_bitstring_type(self, t):
256         return self.inline_simple_type(t)
257
258     def defn_bitstring_type(self, class_name, t):
259         fragment = self.writer.get_fragment()
260
261         if t.named_bits:
262             fragment.write_line('%s.namedValues = namedval.NamedValues(' % class_name)
263             fragment.push_indent()
264             named_bits = ['(\'%s\', %s)' % (b.identifier, b.value) for b in t.named_bits]
265             fragment.write_enumeration(named_bits)
266             fragment.pop_indent()
267             fragment.write_line(')')
268
269         if t.constraint:
270             fragment.write_line('%s.subtypeSpec=%s' % (class_name, self.build_constraint_expr(t.constraint)))
271
272         return str(fragment)
273
274     def defn_collection_type(self, class_name, t):
275         fragment = self.writer.get_fragment()
276         fragment.write_line('%s.componentType = %s' % (class_name, self.generate_expr(t.type_decl)))
277
278         if t.size_constraint:
279             fragment.write_line('%s.subtypeSpec=%s' % (class_name, self.build_constraint_expr(t.size_constraint)))
280
281         return str(fragment)
282
283     def inline_simple_type(self, t):
284         type_expr = _translate_type(t.type_name) + '()'
285         if t.constraint:
286             type_expr += '.subtype(subtypeSpec=%s)' % self.build_constraint_expr(t.constraint)
287
288         return type_expr
289
290     def inline_defined_type(self, t):
291         translated_type = _translate_type(t.type_name) + '()'
292         if t.module_name and t.module_name != self.sema_module.name:
293             translated_type = _sanitize_module(t.module_name) + '.' + translated_type
294         return translated_type 
295
296     def inline_constructed_type(self, t):
297         fragment = self.writer.get_fragment()
298
299         class_name = _translate_type(t.type_name)
300
301         fragment.write_line('%s(componentType=namedtype.NamedTypes(' % class_name)
302
303         fragment.push_indent()
304         fragment.write_block(self.inline_component_types(t.components))
305         fragment.pop_indent()
306
307         fragment.write_line('))')
308
309         return str(fragment)
310
311     def inline_component_types(self, components):
312         fragment = self.writer.get_fragment()
313
314         component_exprs = []
315         for c in components:
316             if not isinstance(c, ExtensionMarker):
317                 component_exprs.append(self.generate_expr(c))
318
319         fragment.write_enumeration(component_exprs)
320
321         return str(fragment)
322
323     def inline_tagged_type(self, t):
324         implicity = self.sema_module.resolve_tag_implicity(t.implicity, t.type_decl)
325         if implicity == TagImplicity.IMPLICIT:
326             tag_implicity = 'implicitTag'
327         elif implicity == TagImplicity.EXPLICIT:
328             tag_implicity = 'explicitTag'
329         else:
330             assert False, "Unexpected implicity: %s" % implicity
331
332         type_expr = self.generate_expr(t.type_decl)
333         type_expr += '.subtype(%s=%s)' % (tag_implicity, self.build_tag_expr(t))
334
335         return type_expr
336
337     def inline_selection_type(self, t):
338         selected_type = self.sema_module.resolve_selection_type(t)
339         assert selected_type is not None, "Found no member %s in %s" % (t.identifier, t.type_decl)
340
341         return self.generate_expr(selected_type)
342
343     def build_tag_expr(self, tag_def):
344         context = _translate_tag_class(tag_def.class_name)
345
346         tagged_type_decl = self.sema_module.resolve_type_decl(tag_def.type_decl, self.referenced_modules)
347         if isinstance(tagged_type_decl, ConstructedType):
348             tag_format = 'tag.tagFormatConstructed'
349         else:
350             tag_format = 'tag.tagFormatSimple'
351
352         return 'tag.Tag(%s, %s, %s)' % (context, tag_format, tag_def.class_number)
353
354     def build_constraint_expr(self, constraint):
355         def unpack_size_constraint(nested):
356             if isinstance(nested, SingleValueConstraint):
357                 return _translate_value(nested.value), _translate_value(nested.value)
358             elif isinstance(nested, ValueRangeConstraint):
359                 return _translate_value(nested.min_value), _translate_value(nested.max_value)
360             else:
361                 assert False, "Unrecognized nested size constraint type: %s" % type(constraint.nested).__name__
362
363         if isinstance(constraint, SingleValueConstraint):
364             return 'constraint.SingleValueConstraint(%s)' % (_translate_value(constraint.value))
365         elif isinstance(constraint, SizeConstraint):
366             min_value, max_value = unpack_size_constraint(constraint.nested)
367             return 'constraint.ValueSizeConstraint(%s, %s)' % (_translate_value(min_value), _translate_value(max_value))
368         elif isinstance (constraint, ValueRangeConstraint):
369             return 'constraint.ValueRangeConstraint(%s, %s)' % (_translate_value(constraint.min_value),
370                                                                 _translate_value(constraint.max_value))
371         else:
372             assert False, "Unrecognized constraint type: %s" % type(constraint).__name__
373
374     def build_value_construct_expr(self, type_decl, value):
375         """ Build a valid construct-expression for values, depending on
376         the target pyasn1 type.
377         """
378
379         def build_value_expr(type_name, value):
380             """ Special treatment for bstring and hstring values,
381             which use different construction depending on target type.
382             """
383             if isinstance(value, BinaryStringValue):
384                 if type_name == 'OCTET STRING':
385                     return 'binValue=\'%s\'' % value.value
386                 else:
387                     return '"\'%s\'B"' % value.value
388             elif isinstance(value, HexStringValue):
389                 if type_name == 'OCTET STRING':
390                     return 'hexValue=\'%s\'' % value.value
391                 else:
392                     return '"\'%s\'H"' % value.value
393             else:
394                 return _translate_value(value)
395
396         if isinstance(value, ObjectIdentifierValue):
397             return self.build_object_identifier_value(value)
398         else:
399             value_type = _translate_type(type_decl.type_name)
400             root_type = self.sema_module.resolve_type_decl(type_decl, self.referenced_modules)
401             return '%s(%s)' % (value_type, build_value_expr(root_type.type_name, value))
402
403     def inline_component_type(self, t):
404         if t.components_of_type:
405             # COMPONENTS OF works like a literal include, so just
406             # expand all components of the referenced type.
407             included_type_decl = self.sema_module.resolve_type_decl(t.components_of_type, self.referenced_modules)
408             included_content = self.inline_component_types(included_type_decl.components)
409
410             # Strip trailing newline from inline_component_types
411             # to make the list line up
412             return included_content.strip()
413
414         if t.optional:
415             return "namedtype.OptionalNamedType('%s', %s)" % (t.identifier, self.generate_expr(t.type_decl))
416         elif t.default_value is not None:
417             type_expr = self.generate_expr(t.type_decl)
418             type_expr += '.subtype(value=%s)' % _translate_value(t.default_value)
419
420             return "namedtype.DefaultedNamedType('%s', %s)" % (t.identifier, type_expr)
421         else:
422             return "namedtype.NamedType('%s', %s)" % (t.identifier, self.generate_expr(t.type_decl))
423
424     def inline_named_type(self, t):
425         return "namedtype.NamedType('%s', %s)" % (t.identifier, self.generate_expr(t.type_decl))
426
427     def inline_value_list_type(self, t):
428         class_name = _translate_type(t.type_name)
429         if t.named_values:
430             named_values = ['(\'%s\', %s)' % (v.identifier, v.value) for v in t.named_values if not isinstance(v, ExtensionMarker)]
431             return '%s(namedValues=namedval.NamedValues(%s))' % (class_name, ', '.join(named_values))
432         else:
433             return class_name + '()'
434
435     def inline_sequenceof_type(self, t):
436         return 'univ.SequenceOf(componentType=%s)' % self.generate_expr(t.type_decl)
437
438     def inline_setof_type(self, t):
439         return 'univ.SetOf(componentType=%s)' % self.generate_expr(t.type_decl)
440
441     def build_object_identifier_value(self, t):
442         objid_components = []
443
444         for c in t.components:
445             if isinstance(c, NameForm):
446                 if c.name in REGISTERED_OID_NAMES:
447                     objid_components.append(str(REGISTERED_OID_NAMES[c.name]))
448                 else:
449                     objid_components.append(_translate_value(c.name))
450             elif isinstance(c, NumberForm):
451                 objid_components.append(str(c.value))
452             elif isinstance(c, NameAndNumberForm):
453                 objid_components.append(str(c.number.value))
454             else:
455                 assert False
456
457         return '_OID(%s)' % ', '.join(objid_components)
458
459     def generate_OID(self):
460         fragment = self.writer.get_fragment()
461
462         fragment.write_line('def _OID(*components):')
463         fragment.push_indent()
464         fragment.write_line('output = []')
465         fragment.write_line('for x in tuple(components):')
466         fragment.push_indent()
467         fragment.write_line('if isinstance(x, univ.ObjectIdentifier):')
468         fragment.push_indent()
469         fragment.write_line('output.extend(list(x))')
470         fragment.pop_indent()
471         fragment.write_line('else:')
472         fragment.push_indent()
473         fragment.write_line('output.append(int(x))')
474         fragment.pop_indent()
475         fragment.pop_indent()
476         fragment.write_blanks(1)
477         fragment.write_line('return univ.ObjectIdentifier(output)')
478         fragment.pop_indent()
479
480         fragment.pop_indent()
481
482         return str(fragment)
483
484
485 def generate_pyasn1(sema_module, out_stream, referenced_modules):
486     return Pyasn1Backend(sema_module, out_stream, referenced_modules).generate_code()
487
488
489 # Translation tables from ASN.1 primitives to pyasn1 primitives
490 _ASN1_TAG_CONTEXTS = {
491     'APPLICATION': 'tag.tagClassApplication',
492     'PRIVATE': 'tag.tagClassPrivate',
493     'UNIVERSAL': 'tag.tagClassUniversal'
494 }
495
496
497 _ASN1_BUILTIN_VALUES = {
498     'FALSE': '0',
499     'TRUE': '1'
500 }
501
502
503 _ASN1_BUILTIN_TYPES = {
504     'ANY': 'univ.Any',
505     'INTEGER': 'univ.Integer',
506     'BOOLEAN': 'univ.Boolean',
507     'NULL': 'univ.Null',
508     'ENUMERATED': 'univ.Enumerated',
509     'REAL': 'univ.Real',
510     'BIT STRING': 'univ.BitString',
511     'OCTET STRING': 'univ.OctetString',
512     'CHOICE': 'univ.Choice',
513     'SEQUENCE': 'univ.Sequence',
514     'SET': 'univ.Set',
515     'SEQUENCE OF': 'univ.SequenceOf',
516     'SET OF': 'univ.SetOf',
517     'OBJECT IDENTIFIER': 'univ.ObjectIdentifier',
518     'UTF8String': 'char.UTF8String',
519     'GeneralString': 'char.GeneralString',
520     'NumericString': 'char.NumericString',
521     'PrintableString': 'char.PrintableString',
522     'IA5String': 'char.IA5String',
523     'GraphicString': 'char.GraphicString',
524     'GeneralizedTime': 'useful.GeneralizedTime',
525     'UTCTime': 'useful.UTCTime',
526     'ObjectDescriptor': 'useful.ObjectDescriptor',  # In pyasn1 r1.2
527     'VisibleString': 'char.VisibleString',
528     'TeletexString': 'char.TeletexString',
529     'UniversalString': 'char.UniversalString',
530     'BMPString': 'char.BMPString',
531     'T61String': 'char.T61String',
532     'VideotexString': 'char.VideotexString',
533 }
534
535
536 def _translate_type(type_name):
537     """ Translate ASN.1 built-in types to pyasn1 equivalents.
538     Non-builtins are not translated.
539     """
540     assert isinstance(type_name, str), "Type name must be a string."
541     type_name = _sanitize_identifier(type_name)
542
543     return _ASN1_BUILTIN_TYPES.get(type_name, type_name)
544
545
546 def _translate_tag_class(tag_class):
547     """ Translate ASN.1 tag class names to pyasn1 equivalents.
548     Defaults to tag.tagClassContext if tag_class is not
549     recognized.
550     """
551     return _ASN1_TAG_CONTEXTS.get(tag_class, 'tag.tagClassContext')
552
553
554 def _translate_value(value):
555     """ Translate ASN.1 built-in values to Python equivalents.
556     Unrecognized values are not translated.
557     """
558     if isinstance(value, ReferencedValue) or _heuristic_is_identifier(value):
559         value = _sanitize_identifier(value)
560
561     return _ASN1_BUILTIN_VALUES.get(value, value)
562
563
564 def _heuristic_is_identifier(value):
565     """ Return True if this value is likely an identifier.
566     """
567     first = str(value)[0]
568     return first != '-' and not first.isdigit()
569
570
571 def _sanitize_identifier(name):
572     """ Sanitize ASN.1 type and value identifiers so that they're
573     valid Python identifiers.
574     """
575     name = str(name)
576     name = name.replace('-', '_')
577     if name in keyword.kwlist:
578         name += '_'
579
580     return name
581
582
583 def _sanitize_module(name):
584     """ Sanitize ASN.1 module identifiers so that they're PEP8 compliant identifiers.
585     """
586     return _sanitize_identifier(name).lower()
587
588 # Simplistic command-line driver
589 def main():
590     arg_parser = argparse.ArgumentParser(description='Generate Python classes from an ASN.1 definition file. Output to stdout by default.')
591     arg_parser.add_argument('file', metavar='file', type=argparse.FileType('r'),
592                             help='the ASN.1 file to process')
593     arg_parser.add_argument('--split', action='store_true',
594                             help='output multiple modules to separate files')
595     args = arg_parser.parse_args()
596     asn1def = args.file.read()
597
598     parse_tree = parser.parse_asn1(asn1def)
599
600     modules = build_semantic_model(parse_tree)
601     if len(modules) > 1 and not args.split:
602         print('WARNING: More than one module generated to the same stream.', file=sys.stderr)
603
604     output_file = sys.stdout
605     for module in modules:
606         try:
607             if args.split:
608                 output_file = open(_sanitize_module(module.name) + '.py', 'w')
609             print(pygen.auto_generated_header(), file=output_file)
610             generate_pyasn1(module, output_file, modules)
611         finally:
612             if output_file != sys.stdout:
613                 output_file.close()
614
615     return 0
616
617
618 if __name__ == '__main__':
619     sys.exit(main())