1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """Classes to handle advanced configuration in simple to complex applications.
19
20 Allows to load the configuration from a file or from command line
21 options, to generate a sample configuration file or to display
22 program's usage. Fills the gap between optik/optparse and ConfigParser
23 by adding data types (which are also available as a standalone optik
24 extension in the `optik_ext` module).
25
26
27 Quick start: simplest usage
28 ---------------------------
29
30 .. python ::
31
32 >>> import sys
33 >>> from logilab.common.configuration import Configuration
34 >>> options = [('dothis', {'type':'yn', 'default': True, 'metavar': '<y or n>'}),
35 ... ('value', {'type': 'string', 'metavar': '<string>'}),
36 ... ('multiple', {'type': 'csv', 'default': ('yop',),
37 ... 'metavar': '<comma separated values>',
38 ... 'help': 'you can also document the option'}),
39 ... ('number', {'type': 'int', 'default':2, 'metavar':'<int>'}),
40 ... ]
41 >>> config = Configuration(options=options, name='My config')
42 >>> print config['dothis']
43 True
44 >>> print config['value']
45 None
46 >>> print config['multiple']
47 ('yop',)
48 >>> print config['number']
49 2
50 >>> print config.help()
51 Usage: [options]
52
53 Options:
54 -h, --help show this help message and exit
55 --dothis=<y or n>
56 --value=<string>
57 --multiple=<comma separated values>
58 you can also document the option [current: none]
59 --number=<int>
60
61 >>> f = open('myconfig.ini', 'w')
62 >>> f.write('''[MY CONFIG]
63 ... number = 3
64 ... dothis = no
65 ... multiple = 1,2,3
66 ... ''')
67 >>> f.close()
68 >>> config.load_file_configuration('myconfig.ini')
69 >>> print config['dothis']
70 False
71 >>> print config['value']
72 None
73 >>> print config['multiple']
74 ['1', '2', '3']
75 >>> print config['number']
76 3
77 >>> sys.argv = ['mon prog', '--value', 'bacon', '--multiple', '4,5,6',
78 ... 'nonoptionargument']
79 >>> print config.load_command_line_configuration()
80 ['nonoptionargument']
81 >>> print config['value']
82 bacon
83 >>> config.generate_config()
84 # class for simple configurations which don't need the
85 # manager / providers model and prefer delegation to inheritance
86 #
87 # configuration values are accessible through a dict like interface
88 #
89 [MY CONFIG]
90
91 dothis=no
92
93 value=bacon
94
95 # you can also document the option
96 multiple=4,5,6
97
98 number=3
99
100 Note : starting with Python 2.7 ConfigParser is able to take into
101 account the order of occurrences of the options into a file (by
102 using an OrderedDict). If you have two options changing some common
103 state, like a 'disable-all-stuff' and a 'enable-some-stuff-a', their
104 order of appearance will be significant : the last specified in the
105 file wins. For earlier version of python and logilab.common newer
106 than 0.61 the behaviour is unspecified.
107
108 """
109
110 from __future__ import print_function
111
112 __docformat__ = "restructuredtext en"
113
114 __all__ = ('OptionsManagerMixIn', 'OptionsProviderMixIn',
115 'ConfigurationMixIn', 'Configuration',
116 'OptionsManager2ConfigurationAdapter')
117
118 import os
119 import sys
120 import re
121 from os.path import exists, expanduser
122 from copy import copy
123 from warnings import warn
124
125 from six import integer_types, string_types
126 from six.moves import range, configparser as cp, input
127
128 from logilab.common.compat import str_encode as _encode
129 from logilab.common.deprecation import deprecated
130 from logilab.common.textutils import normalize_text, unquote
131 from logilab.common import optik_ext
132
133 OptionError = optik_ext.OptionError
134
135 REQUIRED = []
136
138 """raised by set_option when it doesn't know what to do for an action"""
139
140
142 encoding = encoding or getattr(stream, 'encoding', None)
143 if not encoding:
144 import locale
145 encoding = locale.getpreferredencoding()
146 return encoding
147
148
149
150
151
152
153
155 """validate and return a converted value for option of type 'choice'
156 """
157 if not value in optdict['choices']:
158 msg = "option %s: invalid value: %r, should be in %s"
159 raise optik_ext.OptionValueError(msg % (name, value, optdict['choices']))
160 return value
161
163 """validate and return a converted value for option of type 'choice'
164 """
165 choices = optdict['choices']
166 values = optik_ext.check_csv(None, name, value)
167 for value in values:
168 if not value in choices:
169 msg = "option %s: invalid value: %r, should be in %s"
170 raise optik_ext.OptionValueError(msg % (name, value, choices))
171 return values
172
174 """validate and return a converted value for option of type 'csv'
175 """
176 return optik_ext.check_csv(None, name, value)
177
179 """validate and return a converted value for option of type 'yn'
180 """
181 return optik_ext.check_yn(None, name, value)
182
184 """validate and return a converted value for option of type 'named'
185 """
186 return optik_ext.check_named(None, name, value)
187
189 """validate and return a filepath for option of type 'file'"""
190 return optik_ext.check_file(None, name, value)
191
193 """validate and return a valid color for option of type 'color'"""
194 return optik_ext.check_color(None, name, value)
195
197 """validate and return a string for option of type 'password'"""
198 return optik_ext.check_password(None, name, value)
199
201 """validate and return a mx DateTime object for option of type 'date'"""
202 return optik_ext.check_date(None, name, value)
203
205 """validate and return a time object for option of type 'time'"""
206 return optik_ext.check_time(None, name, value)
207
209 """validate and return an integer for option of type 'bytes'"""
210 return optik_ext.check_bytes(None, name, value)
211
212
213 VALIDATORS = {'string': unquote,
214 'int': int,
215 'float': float,
216 'file': file_validator,
217 'font': unquote,
218 'color': color_validator,
219 'regexp': re.compile,
220 'csv': csv_validator,
221 'yn': yn_validator,
222 'bool': yn_validator,
223 'named': named_validator,
224 'password': password_validator,
225 'date': date_validator,
226 'time': time_validator,
227 'bytes': bytes_validator,
228 'choice': choice_validator,
229 'multiple_choice': multiple_choice_validator,
230 }
231
233 if opttype not in VALIDATORS:
234 raise Exception('Unsupported type "%s"' % opttype)
235 try:
236 return VALIDATORS[opttype](optdict, option, value)
237 except TypeError:
238 try:
239 return VALIDATORS[opttype](value)
240 except optik_ext.OptionValueError:
241 raise
242 except:
243 raise optik_ext.OptionValueError('%s value (%r) should be of type %s' %
244 (option, value, opttype))
245
246
247
248
249
250
251
260
264
276 return input_validator
277
278 INPUT_FUNCTIONS = {
279 'string': input_string,
280 'password': input_password,
281 }
282
283 for opttype in VALIDATORS.keys():
284 INPUT_FUNCTIONS.setdefault(opttype, _make_input_function(opttype))
285
286
287
289 """monkey patch OptionParser.expand_default since we have a particular
290 way to handle defaults to avoid overriding values in the configuration
291 file
292 """
293 if self.parser is None or not self.default_tag:
294 return option.help
295 optname = option._long_opts[0][2:]
296 try:
297 provider = self.parser.options_manager._all_options[optname]
298 except KeyError:
299 value = None
300 else:
301 optdict = provider.get_option_def(optname)
302 optname = provider.option_attrname(optname, optdict)
303 value = getattr(provider.config, optname, optdict)
304 value = format_option_value(optdict, value)
305 if value is optik_ext.NO_DEFAULT or not value:
306 value = self.NO_DEFAULT_VALUE
307 return option.help.replace(self.default_tag, str(value))
308
309
311 """return a validated value for an option according to its type
312
313 optional argument name is only used for error message formatting
314 """
315 try:
316 _type = optdict['type']
317 except KeyError:
318
319 return value
320 return _call_validator(_type, optdict, name, value)
321 convert = deprecated('[0.60] convert() was renamed _validate()')(_validate)
322
323
324
329
346
361
380
388
408
409 format_section = ini_format_section
410
429
430
431
433 """MixIn to handle a configuration from both a configuration file and
434 command line options
435 """
436
437 - def __init__(self, usage, config_file=None, version=None, quiet=0):
438 self.config_file = config_file
439 self.reset_parsers(usage, version=version)
440
441 self.options_providers = []
442
443 self._all_options = {}
444 self._short_options = {}
445 self._nocallback_options = {}
446 self._mygroups = dict()
447
448 self.quiet = quiet
449 self._maxlevel = 0
450
452
453 self.cfgfile_parser = cp.ConfigParser()
454
455 self.cmdline_parser = optik_ext.OptionParser(usage=usage, version=version)
456 self.cmdline_parser.options_manager = self
457 self._optik_option_attrs = set(self.cmdline_parser.option_class.ATTRS)
458
460 """register an options provider"""
461 assert provider.priority <= 0, "provider's priority can't be >= 0"
462 for i in range(len(self.options_providers)):
463 if provider.priority > self.options_providers[i].priority:
464 self.options_providers.insert(i, provider)
465 break
466 else:
467 self.options_providers.append(provider)
468 non_group_spec_options = [option for option in provider.options
469 if 'group' not in option[1]]
470 groups = getattr(provider, 'option_groups', ())
471 if own_group and non_group_spec_options:
472 self.add_option_group(provider.name.upper(), provider.__doc__,
473 non_group_spec_options, provider)
474 else:
475 for opt, optdict in non_group_spec_options:
476 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
477 for gname, gdoc in groups:
478 gname = gname.upper()
479 goptions = [option for option in provider.options
480 if option[1].get('group', '').upper() == gname]
481 self.add_option_group(gname, gdoc, goptions, provider)
482
484 """add an option group including the listed options
485 """
486 assert options
487
488 if group_name in self._mygroups:
489 group = self._mygroups[group_name]
490 else:
491 group = optik_ext.OptionGroup(self.cmdline_parser,
492 title=group_name.capitalize())
493 self.cmdline_parser.add_option_group(group)
494 group.level = provider.level
495 self._mygroups[group_name] = group
496
497 if group_name != "DEFAULT":
498 self.cfgfile_parser.add_section(group_name)
499
500 for opt, optdict in options:
501 self.add_optik_option(provider, group, opt, optdict)
502
504 if 'inputlevel' in optdict:
505 warn('[0.50] "inputlevel" in option dictionary for %s is deprecated,'
506 ' use "level"' % opt, DeprecationWarning)
507 optdict['level'] = optdict.pop('inputlevel')
508 args, optdict = self.optik_option(provider, opt, optdict)
509 option = optikcontainer.add_option(*args, **optdict)
510 self._all_options[opt] = provider
511 self._maxlevel = max(self._maxlevel, option.level or 0)
512
514 """get our personal option definition and return a suitable form for
515 use with optik/optparse
516 """
517 optdict = copy(optdict)
518 others = {}
519 if 'action' in optdict:
520 self._nocallback_options[provider] = opt
521 else:
522 optdict['action'] = 'callback'
523 optdict['callback'] = self.cb_set_provider_option
524
525
526 if 'default' in optdict:
527 if ('help' in optdict
528 and optdict.get('default') is not None
529 and not optdict['action'] in ('store_true', 'store_false')):
530 optdict['help'] += ' [current: %default]'
531 del optdict['default']
532 args = ['--' + str(opt)]
533 if 'short' in optdict:
534 self._short_options[optdict['short']] = opt
535 args.append('-' + optdict['short'])
536 del optdict['short']
537
538 for key in list(optdict.keys()):
539 if not key in self._optik_option_attrs:
540 optdict.pop(key)
541 return args, optdict
542
544 """optik callback for option setting"""
545 if opt.startswith('--'):
546
547 opt = opt[2:]
548 else:
549
550 opt = self._short_options[opt[1:]]
551
552 if value is None:
553 value = 1
554 self.global_set_option(opt, value)
555
557 """set option on the correct option provider"""
558 self._all_options[opt].set_option(opt, value)
559
561 """write a configuration file according to the current configuration
562 into the given stream or stdout
563 """
564 options_by_section = {}
565 sections = []
566 for provider in self.options_providers:
567 for section, options in provider.options_by_section():
568 if section is None:
569 section = provider.name
570 if section in skipsections:
571 continue
572 options = [(n, d, v) for (n, d, v) in options
573 if d.get('type') is not None]
574 if not options:
575 continue
576 if not section in sections:
577 sections.append(section)
578 alloptions = options_by_section.setdefault(section, [])
579 alloptions += options
580 stream = stream or sys.stdout
581 encoding = _get_encoding(encoding, stream)
582 printed = False
583 for section in sections:
584 if printed:
585 print('\n', file=stream)
586 format_section(stream, section.upper(), options_by_section[section],
587 encoding)
588 printed = True
589
590 - def generate_manpage(self, pkginfo, section=1, stream=None):
591 """write a man page for the current configuration into the given
592 stream or stdout
593 """
594 self._monkeypatch_expand_default()
595 try:
596 optik_ext.generate_manpage(self.cmdline_parser, pkginfo,
597 section, stream=stream or sys.stdout,
598 level=self._maxlevel)
599 finally:
600 self._unmonkeypatch_expand_default()
601
602
603
605 """initialize configuration using default values"""
606 for provider in self.options_providers:
607 provider.load_defaults()
608
613
615 """read the configuration file but do not load it (i.e. dispatching
616 values to each options provider)
617 """
618 helplevel = 1
619 while helplevel <= self._maxlevel:
620 opt = '-'.join(['long'] * helplevel) + '-help'
621 if opt in self._all_options:
622 break
623 def helpfunc(option, opt, val, p, level=helplevel):
624 print(self.help(level))
625 sys.exit(0)
626 helpmsg = '%s verbose help.' % ' '.join(['more'] * helplevel)
627 optdict = {'action' : 'callback', 'callback' : helpfunc,
628 'help' : helpmsg}
629 provider = self.options_providers[0]
630 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
631 provider.options += ( (opt, optdict), )
632 helplevel += 1
633 if config_file is None:
634 config_file = self.config_file
635 if config_file is not None:
636 config_file = expanduser(config_file)
637 if config_file and exists(config_file):
638 parser = self.cfgfile_parser
639 parser.read([config_file])
640
641 for sect, values in list(parser._sections.items()):
642 if not sect.isupper() and values:
643 parser._sections[sect.upper()] = values
644 elif not self.quiet:
645 msg = 'No config file found, using default configuration'
646 print(msg, file=sys.stderr)
647 return
648
666
668 """dispatch values previously read from a configuration file to each
669 options provider)
670 """
671 parser = self.cfgfile_parser
672 for section in parser.sections():
673 for option, value in parser.items(section):
674 try:
675 self.global_set_option(option, value)
676 except (KeyError, OptionError):
677
678 continue
679
681 """override configuration according to given parameters
682 """
683 for opt, opt_value in kwargs.items():
684 opt = opt.replace('_', '-')
685 provider = self._all_options[opt]
686 provider.set_option(opt, opt_value)
687
689 """override configuration according to command line parameters
690
691 return additional arguments
692 """
693 self._monkeypatch_expand_default()
694 try:
695 if args is None:
696 args = sys.argv[1:]
697 else:
698 args = list(args)
699 (options, args) = self.cmdline_parser.parse_args(args=args)
700 for provider in self._nocallback_options.keys():
701 config = provider.config
702 for attr in config.__dict__.keys():
703 value = getattr(options, attr, None)
704 if value is None:
705 continue
706 setattr(config, attr, value)
707 return args
708 finally:
709 self._unmonkeypatch_expand_default()
710
711
712
713
722
724
725 try:
726 self.__expand_default_backup = optik_ext.HelpFormatter.expand_default
727 optik_ext.HelpFormatter.expand_default = expand_default
728 except AttributeError:
729
730 pass
732
733 if hasattr(optik_ext.HelpFormatter, 'expand_default'):
734
735 optik_ext.HelpFormatter.expand_default = self.__expand_default_backup
736
737 - def help(self, level=0):
738 """return the usage string for available options """
739 self.cmdline_parser.formatter.output_level = level
740 self._monkeypatch_expand_default()
741 try:
742 return self.cmdline_parser.format_help()
743 finally:
744 self._unmonkeypatch_expand_default()
745
746
748 """used to ease late binding of default method (so you can define options
749 on the class using default methods on the configuration instance)
750 """
752 self.method = methname
753 self._inst = None
754
755 - def bind(self, instance):
756 """bind the method to its instance"""
757 if self._inst is None:
758 self._inst = instance
759
761 assert self._inst, 'unbound method'
762 return getattr(self._inst, self.method)(*args, **kwargs)
763
764
765
767 """Mixin to provide options to an OptionsManager"""
768
769
770 priority = -1
771 name = 'default'
772 options = ()
773 level = 0
774
776 self.config = optik_ext.Values()
777 for option in self.options:
778 try:
779 option, optdict = option
780 except ValueError:
781 raise Exception('Bad option: %r' % option)
782 if isinstance(optdict.get('default'), Method):
783 optdict['default'].bind(self)
784 elif isinstance(optdict.get('callback'), Method):
785 optdict['callback'].bind(self)
786 self.load_defaults()
787
789 """initialize the provider using default values"""
790 for opt, optdict in self.options:
791 action = optdict.get('action')
792 if action != 'callback':
793
794 default = self.option_default(opt, optdict)
795 if default is REQUIRED:
796 continue
797 self.set_option(opt, default, action, optdict)
798
800 """return the default value for an option"""
801 if optdict is None:
802 optdict = self.get_option_def(opt)
803 default = optdict.get('default')
804 if callable(default):
805 default = default()
806 return default
807
809 """get the config attribute corresponding to opt
810 """
811 if optdict is None:
812 optdict = self.get_option_def(opt)
813 return optdict.get('dest', opt.replace('-', '_'))
814 option_name = deprecated('[0.60] OptionsProviderMixIn.option_name() was renamed to option_attrname()')(option_attrname)
815
817 """get the current value for the given option"""
818 return getattr(self.config, self.option_attrname(opt), None)
819
820 - def set_option(self, opt, value, action=None, optdict=None):
821 """method called to set an option (registered in the options list)
822 """
823 if optdict is None:
824 optdict = self.get_option_def(opt)
825 if value is not None:
826 value = _validate(value, optdict, opt)
827 if action is None:
828 action = optdict.get('action', 'store')
829 if optdict.get('type') == 'named':
830 optname = self.option_attrname(opt, optdict)
831 currentvalue = getattr(self.config, optname, None)
832 if currentvalue:
833 currentvalue.update(value)
834 value = currentvalue
835 if action == 'store':
836 setattr(self.config, self.option_attrname(opt, optdict), value)
837 elif action in ('store_true', 'count'):
838 setattr(self.config, self.option_attrname(opt, optdict), 0)
839 elif action == 'store_false':
840 setattr(self.config, self.option_attrname(opt, optdict), 1)
841 elif action == 'append':
842 opt = self.option_attrname(opt, optdict)
843 _list = getattr(self.config, opt, None)
844 if _list is None:
845 if isinstance(value, (list, tuple)):
846 _list = value
847 elif value is not None:
848 _list = []
849 _list.append(value)
850 setattr(self.config, opt, _list)
851 elif isinstance(_list, tuple):
852 setattr(self.config, opt, _list + (value,))
853 else:
854 _list.append(value)
855 elif action == 'callback':
856 optdict['callback'](None, opt, value, None)
857 else:
858 raise UnsupportedAction(action)
859
880
882 """return the dictionary defining an option given it's name"""
883 assert self.options
884 for option in self.options:
885 if option[0] == opt:
886 return option[1]
887 raise OptionError('no such option %s in section %r'
888 % (opt, self.name), opt)
889
890
892 """return an iterator on available options for this provider
893 option are actually described by a 3-uple:
894 (section, option name, option dictionary)
895 """
896 for section, options in self.options_by_section():
897 if section is None:
898 if self.name is None:
899 continue
900 section = self.name.upper()
901 for option, optiondict, value in options:
902 yield section, option, optiondict
903
905 """return an iterator on options grouped by section
906
907 (section, [list of (optname, optdict, optvalue)])
908 """
909 sections = {}
910 for optname, optdict in self.options:
911 sections.setdefault(optdict.get('group'), []).append(
912 (optname, optdict, self.option_value(optname)))
913 if None in sections:
914 yield None, sections.pop(None)
915 for section, options in sorted(sections.items()):
916 yield section.upper(), options
917
923
924
925
927 """basic mixin for simple configurations which don't need the
928 manager / providers model
929 """
946
955
958
960 return iter(self.config.__dict__.items())
961
963 try:
964 return getattr(self.config, self.option_attrname(key))
965 except (optik_ext.OptionValueError, AttributeError):
966 raise KeyError(key)
967
970
971 - def get(self, key, default=None):
976
977
979 """class for simple configurations which don't need the
980 manager / providers model and prefer delegation to inheritance
981
982 configuration values are accessible through a dict like interface
983 """
984
985 - def __init__(self, config_file=None, options=None, name=None,
986 usage=None, doc=None, version=None):
994
995
997 """Adapt an option manager to behave like a
998 `logilab.common.configuration.Configuration` instance
999 """
1001 self.config = provider
1002
1004 return getattr(self.config, key)
1005
1007 provider = self.config._all_options[key]
1008 try:
1009 return getattr(provider.config, provider.option_attrname(key))
1010 except AttributeError:
1011 raise KeyError(key)
1012
1015
1016 - def get(self, key, default=None):
1017 provider = self.config._all_options[key]
1018 try:
1019 return getattr(provider.config, provider.option_attrname(key))
1020 except AttributeError:
1021 return default
1022
1023
1024
1026 """initialize newconfig from a deprecated configuration file
1027
1028 possible changes:
1029 * ('renamed', oldname, newname)
1030 * ('moved', option, oldgroup, newgroup)
1031 * ('typechanged', option, oldtype, newvalue)
1032 """
1033
1034 changesindex = {}
1035 for action in changes:
1036 if action[0] == 'moved':
1037 option, oldgroup, newgroup = action[1:]
1038 changesindex.setdefault(option, []).append((action[0], oldgroup, newgroup))
1039 continue
1040 if action[0] == 'renamed':
1041 oldname, newname = action[1:]
1042 changesindex.setdefault(newname, []).append((action[0], oldname))
1043 continue
1044 if action[0] == 'typechanged':
1045 option, oldtype, newvalue = action[1:]
1046 changesindex.setdefault(option, []).append((action[0], oldtype, newvalue))
1047 continue
1048 if action[1] in ('added', 'removed'):
1049 continue
1050 raise Exception('unknown change %s' % action[0])
1051
1052 options = []
1053 for optname, optdef in newconfig.options:
1054 for action in changesindex.pop(optname, ()):
1055 if action[0] == 'moved':
1056 oldgroup, newgroup = action[1:]
1057 optdef = optdef.copy()
1058 optdef['group'] = oldgroup
1059 elif action[0] == 'renamed':
1060 optname = action[1]
1061 elif action[0] == 'typechanged':
1062 oldtype = action[1]
1063 optdef = optdef.copy()
1064 optdef['type'] = oldtype
1065 options.append((optname, optdef))
1066 if changesindex:
1067 raise Exception('unapplied changes: %s' % changesindex)
1068 oldconfig = Configuration(options=options, name=newconfig.name)
1069
1070 oldconfig.load_file_configuration(configfile)
1071
1072 changes.reverse()
1073 done = set()
1074 for action in changes:
1075 if action[0] == 'renamed':
1076 oldname, newname = action[1:]
1077 newconfig[newname] = oldconfig[oldname]
1078 done.add(newname)
1079 elif action[0] == 'typechanged':
1080 optname, oldtype, newvalue = action[1:]
1081 newconfig[optname] = newvalue
1082 done.add(optname)
1083 for optname, optdef in newconfig.options:
1084 if optdef.get('type') and not optname in done:
1085 newconfig.set_option(optname, oldconfig[optname], optdict=optdef)
1086
1087
1089 """preprocess a list of options and remove duplicates, returning a new list
1090 (tuple actually) of options.
1091
1092 Options dictionaries are copied to avoid later side-effect. Also, if
1093 `otpgroup` argument is specified, ensure all options are in the given group.
1094 """
1095 alloptions = {}
1096 options = list(options)
1097 for i in range(len(options)-1, -1, -1):
1098 optname, optdict = options[i]
1099 if optname in alloptions:
1100 options.pop(i)
1101 alloptions[optname].update(optdict)
1102 else:
1103 optdict = optdict.copy()
1104 options[i] = (optname, optdict)
1105 alloptions[optname] = optdict
1106 if optgroup is not None:
1107 alloptions[optname]['group'] = optgroup
1108 return tuple(options)
1109