4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Configuration management for Ganeti
24 This module provides the interface to the Ganeti cluster configuration.
26 The configuration data is stored on every node but is updated on the master
27 only. After each update, the master distributes the data to the other nodes.
29 Currently, the data storage format is JSON. YAML was slow and consuming too
39 from ganeti import errors
40 from ganeti import locking
41 from ganeti import utils
42 from ganeti import constants
43 from ganeti import rpc
44 from ganeti import objects
45 from ganeti import serializer
48 _config_lock = locking.SharedLock()
51 def _ValidateConfig(data):
52 if data.version != constants.CONFIG_VERSION:
53 raise errors.ConfigurationError("Cluster configuration version"
54 " mismatch, got %s instead of %s" %
56 constants.CONFIG_VERSION))
60 """The interface to the cluster configuration.
63 def __init__(self, cfg_file=None, offline=False):
65 self._lock = _config_lock
66 self._config_data = None
67 self._offline = offline
69 self._cfg_file = constants.CLUSTER_CONF_FILE
71 self._cfg_file = cfg_file
72 self._temporary_ids = set()
73 self._temporary_drbds = {}
74 # Note: in order to prevent errors when resolving our name in
75 # _DistributeConfig, we compute it here once and reuse it; it's
76 # better to raise an error before starting to modify the config
77 # file than after it was modified
78 self._my_hostname = utils.HostInfo().name
81 # this method needs to be static, so that we can call it on the class
84 """Check if the cluster is configured.
87 return os.path.exists(constants.CLUSTER_CONF_FILE)
89 @locking.ssynchronized(_config_lock, shared=1)
90 def GenerateMAC(self):
91 """Generate a MAC for an instance.
93 This should check the current instances for duplicates.
96 prefix = self._config_data.cluster.mac_prefix
97 all_macs = self._AllMACs()
100 byte1 = random.randrange(0, 256)
101 byte2 = random.randrange(0, 256)
102 byte3 = random.randrange(0, 256)
103 mac = "%s:%02x:%02x:%02x" % (prefix, byte1, byte2, byte3)
104 if mac not in all_macs:
108 raise errors.ConfigurationError("Can't generate unique MAC")
111 @locking.ssynchronized(_config_lock, shared=1)
112 def IsMacInUse(self, mac):
113 """Predicate: check if the specified MAC is in use in the Ganeti cluster.
115 This only checks instances managed by this cluster, it does not
116 check for potential collisions elsewhere.
119 all_macs = self._AllMACs()
120 return mac in all_macs
122 @locking.ssynchronized(_config_lock, shared=1)
123 def GenerateDRBDSecret(self):
124 """Generate a DRBD secret.
126 This checks the current disks for duplicates.
129 all_secrets = self._AllDRBDSecrets()
132 secret = utils.GenerateSecret()
133 if secret not in all_secrets:
137 raise errors.ConfigurationError("Can't generate unique DRBD secret")
140 def _ComputeAllLVs(self):
141 """Compute the list of all LVs.
145 for instance in self._config_data.instances.values():
146 node_data = instance.MapLVsByNode()
147 for lv_list in node_data.values():
148 lvnames.update(lv_list)
151 @locking.ssynchronized(_config_lock, shared=1)
152 def GenerateUniqueID(self, exceptions=None):
153 """Generate an unique disk name.
155 This checks the current node, instances and disk names for
159 - exceptions: a list with some other names which should be checked
160 for uniqueness (used for example when you want to get
161 more than one id at one time without adding each one in
162 turn to the config file
164 Returns: the unique id as a string
168 existing.update(self._temporary_ids)
169 existing.update(self._ComputeAllLVs())
170 existing.update(self._config_data.instances.keys())
171 existing.update(self._config_data.nodes.keys())
172 if exceptions is not None:
173 existing.update(exceptions)
176 unique_id = utils.NewUUID()
177 if unique_id not in existing and unique_id is not None:
180 raise errors.ConfigurationError("Not able generate an unique ID"
181 " (last tried ID: %s" % unique_id)
182 self._temporary_ids.add(unique_id)
186 """Return all MACs present in the config.
190 for instance in self._config_data.instances.values():
191 for nic in instance.nics:
192 result.append(nic.mac)
196 def _AllDRBDSecrets(self):
197 """Return all DRBD secrets present in the config.
200 def helper(disk, result):
201 """Recursively gather secrets from this disk."""
202 if disk.dev_type == constants.DT_DRBD8:
203 result.append(disk.logical_id[5])
205 for child in disk.children:
206 helper(child, result)
209 for instance in self._config_data.instances.values():
210 for disk in instance.disks:
215 @locking.ssynchronized(_config_lock, shared=1)
216 def VerifyConfig(self):
217 """Stub verify function.
222 data = self._config_data
223 for instance_name in data.instances:
224 instance = data.instances[instance_name]
225 if instance.primary_node not in data.nodes:
226 result.append("instance '%s' has invalid primary node '%s'" %
227 (instance_name, instance.primary_node))
228 for snode in instance.secondary_nodes:
229 if snode not in data.nodes:
230 result.append("instance '%s' has invalid secondary node '%s'" %
231 (instance_name, snode))
232 for idx, nic in enumerate(instance.nics):
233 if nic.mac in seen_macs:
234 result.append("instance '%s' has NIC %d mac %s duplicate" %
235 (instance_name, idx, nic.mac))
237 seen_macs.append(nic.mac)
239 # gather the drbd ports for duplicate checks
240 for dsk in instance.disks:
241 if dsk.dev_type in constants.LDS_DRBD:
242 tcp_port = dsk.logical_id[2]
243 if tcp_port not in ports:
245 ports[tcp_port].append((instance.name, "drbd disk %s" % dsk.iv_name))
246 # gather network port reservation
247 net_port = getattr(instance, "network_port", None)
248 if net_port is not None:
249 if net_port not in ports:
251 ports[net_port].append((instance.name, "network port"))
253 # cluster-wide pool of free ports
254 for free_port in self._config_data.cluster.tcpudp_port_pool:
255 if free_port not in ports:
256 ports[free_port] = []
257 ports[free_port].append(("cluster", "port marked as free"))
259 # compute tcp/udp duplicate ports
265 txt = ", ".join(["%s/%s" % val for val in pdata])
266 result.append("tcp/udp port %s has duplicates: %s" % (pnum, txt))
268 # highest used tcp port check
270 if keys[-1] > self._config_data.cluster.highest_used_port:
271 result.append("Highest used port mismatch, saved %s, computed %s" %
272 (self._config_data.cluster.highest_used_port,
277 def _UnlockedSetDiskID(self, disk, node_name):
278 """Convert the unique ID to the ID needed on the target nodes.
280 This is used only for drbd, which needs ip/port configuration.
282 The routine descends down and updates its children also, because
283 this helps when the only the top device is passed to the remote
286 This function is for internal use, when the config lock is already held.
290 for child in disk.children:
291 self._UnlockedSetDiskID(child, node_name)
293 if disk.logical_id is None and disk.physical_id is not None:
295 if disk.dev_type == constants.LD_DRBD8:
296 pnode, snode, port, pminor, sminor, secret = disk.logical_id
297 if node_name not in (pnode, snode):
298 raise errors.ConfigurationError("DRBD device not knowing node %s" %
300 pnode_info = self._UnlockedGetNodeInfo(pnode)
301 snode_info = self._UnlockedGetNodeInfo(snode)
302 if pnode_info is None or snode_info is None:
303 raise errors.ConfigurationError("Can't find primary or secondary node"
304 " for %s" % str(disk))
305 p_data = (pnode_info.secondary_ip, port)
306 s_data = (snode_info.secondary_ip, port)
307 if pnode == node_name:
308 disk.physical_id = p_data + s_data + (pminor, secret)
309 else: # it must be secondary, we tested above
310 disk.physical_id = s_data + p_data + (sminor, secret)
312 disk.physical_id = disk.logical_id
315 @locking.ssynchronized(_config_lock)
316 def SetDiskID(self, disk, node_name):
317 """Convert the unique ID to the ID needed on the target nodes.
319 This is used only for drbd, which needs ip/port configuration.
321 The routine descends down and updates its children also, because
322 this helps when the only the top device is passed to the remote
326 return self._UnlockedSetDiskID(disk, node_name)
328 @locking.ssynchronized(_config_lock)
329 def AddTcpUdpPort(self, port):
330 """Adds a new port to the available port pool.
333 if not isinstance(port, int):
334 raise errors.ProgrammerError("Invalid type passed for port")
336 self._config_data.cluster.tcpudp_port_pool.add(port)
339 @locking.ssynchronized(_config_lock, shared=1)
340 def GetPortList(self):
341 """Returns a copy of the current port list.
344 return self._config_data.cluster.tcpudp_port_pool.copy()
346 @locking.ssynchronized(_config_lock)
347 def AllocatePort(self):
350 The port will be taken from the available port pool or from the
351 default port range (and in this case we increase
355 # If there are TCP/IP ports configured, we use them first.
356 if self._config_data.cluster.tcpudp_port_pool:
357 port = self._config_data.cluster.tcpudp_port_pool.pop()
359 port = self._config_data.cluster.highest_used_port + 1
360 if port >= constants.LAST_DRBD_PORT:
361 raise errors.ConfigurationError("The highest used port is greater"
362 " than %s. Aborting." %
363 constants.LAST_DRBD_PORT)
364 self._config_data.cluster.highest_used_port = port
369 def _ComputeDRBDMap(self, instance):
370 """Compute the used DRBD minor/nodes.
372 Return: dictionary of node_name: dict of minor: instance_name. The
373 returned dict will have all the nodes in it (even if with an empty
377 def _AppendUsedPorts(instance_name, disk, used):
378 if disk.dev_type == constants.LD_DRBD8 and len(disk.logical_id) >= 5:
379 nodeA, nodeB, dummy, minorA, minorB = disk.logical_id[:5]
380 for node, port in ((nodeA, minorA), (nodeB, minorB)):
381 assert node in used, "Instance node not found in node list"
382 if port in used[node]:
383 raise errors.ProgrammerError("DRBD minor already used:"
385 (node, port, instance_name,
388 used[node][port] = instance_name
390 for child in disk.children:
391 _AppendUsedPorts(instance_name, child, used)
393 my_dict = dict((node, {}) for node in self._config_data.nodes)
394 for (node, minor), instance in self._temporary_drbds.iteritems():
395 my_dict[node][minor] = instance
396 for instance in self._config_data.instances.itervalues():
397 for disk in instance.disks:
398 _AppendUsedPorts(instance.name, disk, my_dict)
401 @locking.ssynchronized(_config_lock)
402 def AllocateDRBDMinor(self, nodes, instance):
403 """Allocate a drbd minor.
405 The free minor will be automatically computed from the existing
406 devices. A node can be given multiple times in order to allocate
407 multiple minors. The result is the list of minors, in the same
408 order as the passed nodes.
411 d_map = self._ComputeDRBDMap(instance)
416 # no minors used, we can start at 0
419 self._temporary_drbds[(nname, 0)] = instance
423 ffree = utils.FirstFree(keys)
425 # return the next minor
426 # TODO: implement high-limit check
431 ndata[minor] = instance
432 assert (nname, minor) not in self._temporary_drbds, \
433 "Attempt to reuse reserved DRBD minor"
434 self._temporary_drbds[(nname, minor)] = instance
435 logging.debug("Request to allocate drbd minors, input: %s, returning %s",
439 @locking.ssynchronized(_config_lock)
440 def ReleaseDRBDMinors(self, instance):
441 """Release temporary drbd minors allocated for a given instance.
443 This should be called on both the error paths and on the success
444 paths (after the instance has been added or updated).
446 @type instance: string
447 @param instance: the instance for which temporary minors should be
451 for key, name in self._temporary_drbds.items():
453 del self._temporary_drbds[key]
455 @locking.ssynchronized(_config_lock, shared=1)
456 def GetConfigVersion(self):
457 """Get the configuration version.
459 @return: Config version
462 return self._config_data.version
464 @locking.ssynchronized(_config_lock, shared=1)
465 def GetClusterName(self):
468 @return: Cluster name
471 return self._config_data.cluster.cluster_name
473 @locking.ssynchronized(_config_lock, shared=1)
474 def GetMasterNode(self):
475 """Get the hostname of the master node for this cluster.
477 @return: Master hostname
480 return self._config_data.cluster.master_node
482 @locking.ssynchronized(_config_lock, shared=1)
483 def GetMasterIP(self):
484 """Get the IP of the master node for this cluster.
489 return self._config_data.cluster.master_ip
491 @locking.ssynchronized(_config_lock, shared=1)
492 def GetMasterNetdev(self):
493 """Get the master network device for this cluster.
496 return self._config_data.cluster.master_netdev
498 @locking.ssynchronized(_config_lock, shared=1)
499 def GetFileStorageDir(self):
500 """Get the file storage dir for this cluster.
503 return self._config_data.cluster.file_storage_dir
505 @locking.ssynchronized(_config_lock, shared=1)
506 def GetHypervisorType(self):
507 """Get the hypervisor type for this cluster.
510 return self._config_data.cluster.hypervisor
512 @locking.ssynchronized(_config_lock, shared=1)
513 def GetHostKey(self):
514 """Return the rsa hostkey from the config.
520 return self._config_data.cluster.rsahostkeypub
522 @locking.ssynchronized(_config_lock)
523 def AddInstance(self, instance):
524 """Add an instance to the config.
526 This should be used after creating a new instance.
529 instance: the instance object
531 if not isinstance(instance, objects.Instance):
532 raise errors.ProgrammerError("Invalid type passed to AddInstance")
534 if instance.disk_template != constants.DT_DISKLESS:
535 all_lvs = instance.MapLVsByNode()
536 logging.info("Instance '%s' DISK_LAYOUT: %s", instance.name, all_lvs)
538 instance.serial_no = 1
539 self._config_data.instances[instance.name] = instance
540 self._config_data.cluster.serial_no += 1
543 def _SetInstanceStatus(self, instance_name, status):
544 """Set the instance's status to a given value.
547 if status not in ("up", "down"):
548 raise errors.ProgrammerError("Invalid status '%s' passed to"
549 " ConfigWriter._SetInstanceStatus()" %
552 if instance_name not in self._config_data.instances:
553 raise errors.ConfigurationError("Unknown instance '%s'" %
555 instance = self._config_data.instances[instance_name]
556 if instance.status != status:
557 instance.status = status
558 instance.serial_no += 1
561 @locking.ssynchronized(_config_lock)
562 def MarkInstanceUp(self, instance_name):
563 """Mark the instance status to up in the config.
566 self._SetInstanceStatus(instance_name, "up")
568 @locking.ssynchronized(_config_lock)
569 def RemoveInstance(self, instance_name):
570 """Remove the instance from the configuration.
573 if instance_name not in self._config_data.instances:
574 raise errors.ConfigurationError("Unknown instance '%s'" % instance_name)
575 del self._config_data.instances[instance_name]
576 self._config_data.cluster.serial_no += 1
579 @locking.ssynchronized(_config_lock)
580 def RenameInstance(self, old_name, new_name):
581 """Rename an instance.
583 This needs to be done in ConfigWriter and not by RemoveInstance
584 combined with AddInstance as only we can guarantee an atomic
588 if old_name not in self._config_data.instances:
589 raise errors.ConfigurationError("Unknown instance '%s'" % old_name)
590 inst = self._config_data.instances[old_name]
591 del self._config_data.instances[old_name]
594 for disk in inst.disks:
595 if disk.dev_type == constants.LD_FILE:
596 # rename the file paths in logical and physical id
597 file_storage_dir = os.path.dirname(os.path.dirname(disk.logical_id[1]))
598 disk.physical_id = disk.logical_id = (disk.logical_id[0],
599 os.path.join(file_storage_dir,
603 self._config_data.instances[inst.name] = inst
604 self._config_data.cluster.serial_no += 1
607 @locking.ssynchronized(_config_lock)
608 def MarkInstanceDown(self, instance_name):
609 """Mark the status of an instance to down in the configuration.
612 self._SetInstanceStatus(instance_name, "down")
614 def _UnlockedGetInstanceList(self):
615 """Get the list of instances.
617 This function is for internal use, when the config lock is already held.
620 return self._config_data.instances.keys()
622 @locking.ssynchronized(_config_lock, shared=1)
623 def GetInstanceList(self):
624 """Get the list of instances.
627 array of instances, ex. ['instance2.example.com','instance1.example.com']
628 these contains all the instances, also the ones in Admin_down state
631 return self._UnlockedGetInstanceList()
633 @locking.ssynchronized(_config_lock, shared=1)
634 def ExpandInstanceName(self, short_name):
635 """Attempt to expand an incomplete instance name.
638 return utils.MatchNameComponent(short_name,
639 self._config_data.instances.keys())
641 def _UnlockedGetInstanceInfo(self, instance_name):
642 """Returns informations about an instance.
644 This function is for internal use, when the config lock is already held.
647 if instance_name not in self._config_data.instances:
650 return self._config_data.instances[instance_name]
652 @locking.ssynchronized(_config_lock, shared=1)
653 def GetInstanceInfo(self, instance_name):
654 """Returns informations about an instance.
656 It takes the information from the configuration file. Other informations of
657 an instance are taken from the live systems.
660 instance: name of the instance, ex instance1.example.com
666 return self._UnlockedGetInstanceInfo(instance_name)
668 @locking.ssynchronized(_config_lock, shared=1)
669 def GetAllInstancesInfo(self):
670 """Get the configuration of all instances.
673 @returns: dict of (instance, instance_info), where instance_info is what
674 would GetInstanceInfo return for the node
677 my_dict = dict([(instance, self._UnlockedGetInstanceInfo(instance))
678 for instance in self._UnlockedGetInstanceList()])
681 @locking.ssynchronized(_config_lock)
682 def AddNode(self, node):
683 """Add a node to the configuration.
686 node: an object.Node instance
689 logging.info("Adding node %s to configuration" % node.name)
692 self._config_data.nodes[node.name] = node
693 self._config_data.cluster.serial_no += 1
696 @locking.ssynchronized(_config_lock)
697 def RemoveNode(self, node_name):
698 """Remove a node from the configuration.
701 logging.info("Removing node %s from configuration" % node_name)
703 if node_name not in self._config_data.nodes:
704 raise errors.ConfigurationError("Unknown node '%s'" % node_name)
706 del self._config_data.nodes[node_name]
707 self._config_data.cluster.serial_no += 1
710 @locking.ssynchronized(_config_lock, shared=1)
711 def ExpandNodeName(self, short_name):
712 """Attempt to expand an incomplete instance name.
715 return utils.MatchNameComponent(short_name,
716 self._config_data.nodes.keys())
718 def _UnlockedGetNodeInfo(self, node_name):
719 """Get the configuration of a node, as stored in the config.
721 This function is for internal use, when the config lock is already held.
723 Args: node: nodename (tuple) of the node
725 Returns: the node object
728 if node_name not in self._config_data.nodes:
731 return self._config_data.nodes[node_name]
734 @locking.ssynchronized(_config_lock, shared=1)
735 def GetNodeInfo(self, node_name):
736 """Get the configuration of a node, as stored in the config.
738 Args: node: nodename (tuple) of the node
740 Returns: the node object
743 return self._UnlockedGetNodeInfo(node_name)
745 def _UnlockedGetNodeList(self):
746 """Return the list of nodes which are in the configuration.
748 This function is for internal use, when the config lock is already held.
751 return self._config_data.nodes.keys()
754 @locking.ssynchronized(_config_lock, shared=1)
755 def GetNodeList(self):
756 """Return the list of nodes which are in the configuration.
759 return self._UnlockedGetNodeList()
761 @locking.ssynchronized(_config_lock, shared=1)
762 def GetAllNodesInfo(self):
763 """Get the configuration of all nodes.
766 @returns: dict of (node, node_info), where node_info is what
767 would GetNodeInfo return for the node
770 my_dict = dict([(node, self._UnlockedGetNodeInfo(node))
771 for node in self._UnlockedGetNodeList()])
774 def _BumpSerialNo(self):
775 """Bump up the serial number of the config.
778 self._config_data.serial_no += 1
780 def _OpenConfig(self):
781 """Read the config data from disk.
783 In case we already have configuration data and the config file has
784 the same mtime as when we read it, we skip the parsing of the
785 file, since de-serialisation could be slow.
788 f = open(self._cfg_file, 'r')
791 data = objects.ConfigData.FromDict(serializer.Load(f.read()))
792 except Exception, err:
793 raise errors.ConfigurationError(err)
797 # Make sure the configuration has the right version
798 _ValidateConfig(data)
800 if (not hasattr(data, 'cluster') or
801 not hasattr(data.cluster, 'rsahostkeypub')):
802 raise errors.ConfigurationError("Incomplete configuration"
803 " (missing cluster.rsahostkeypub)")
804 self._config_data = data
806 def _DistributeConfig(self):
807 """Distribute the configuration to the other nodes.
809 Currently, this only copies the configuration file. In the future,
810 it could be used to encapsulate the 2/3-phase update mechanism.
816 nodelist = self._UnlockedGetNodeList()
817 myhostname = self._my_hostname
820 nodelist.remove(myhostname)
824 result = rpc.RpcRunner.call_upload_file(nodelist, self._cfg_file)
825 for node in nodelist:
827 logging.error("copy of file %s to node %s failed",
828 self._cfg_file, node)
832 def _WriteConfig(self, destination=None):
833 """Write the configuration data to persistent storage.
836 if destination is None:
837 destination = self._cfg_file
839 txt = serializer.Dump(self._config_data.ToDict())
840 dir_name, file_name = os.path.split(destination)
841 fd, name = tempfile.mkstemp('.newconfig', file_name, dir_name)
842 f = os.fdopen(fd, 'w')
848 # we don't need to do os.close(fd) as f.close() did it
849 os.rename(name, destination)
850 self.write_count += 1
852 # and redistribute the config file
853 self._DistributeConfig()
855 @locking.ssynchronized(_config_lock)
856 def InitConfig(self, version, cluster_config, master_node_config):
857 """Create the initial cluster configuration.
859 It will contain the current node, which will also be the master
860 node, and no instances.
863 @param version: Configuration version
864 @type cluster_config: objects.Cluster
865 @param cluster_config: Cluster configuration
866 @type master_node_config: objects.Node
867 @param master_node_config: Master node configuration
871 master_node_config.name: master_node_config,
874 self._config_data = objects.ConfigData(version=version,
875 cluster=cluster_config,
881 @locking.ssynchronized(_config_lock, shared=1)
883 """Return the volume group name.
886 return self._config_data.cluster.volume_group_name
888 @locking.ssynchronized(_config_lock)
889 def SetVGName(self, vg_name):
890 """Set the volume group name.
893 self._config_data.cluster.volume_group_name = vg_name
894 self._config_data.cluster.serial_no += 1
897 @locking.ssynchronized(_config_lock, shared=1)
898 def GetDefBridge(self):
899 """Return the default bridge.
902 return self._config_data.cluster.default_bridge
904 @locking.ssynchronized(_config_lock, shared=1)
905 def GetMACPrefix(self):
906 """Return the mac prefix.
909 return self._config_data.cluster.mac_prefix
911 @locking.ssynchronized(_config_lock, shared=1)
912 def GetClusterInfo(self):
913 """Returns informations about the cluster
919 return self._config_data.cluster
921 @locking.ssynchronized(_config_lock)
922 def Update(self, target):
923 """Notify function to be called after updates.
925 This function must be called when an object (as returned by
926 GetInstanceInfo, GetNodeInfo, GetCluster) has been updated and the
927 caller wants the modifications saved to the backing store. Note
928 that all modified objects will be saved, but the target argument
929 is the one the caller wants to ensure that it's saved.
932 if self._config_data is None:
933 raise errors.ProgrammerError("Configuration file not read,"
935 if isinstance(target, objects.Cluster):
936 test = target == self._config_data.cluster
937 elif isinstance(target, objects.Node):
938 test = target in self._config_data.nodes.values()
939 elif isinstance(target, objects.Instance):
940 test = target in self._config_data.instances.values()
942 raise errors.ProgrammerError("Invalid object type (%s) passed to"
943 " ConfigWriter.Update" % type(target))
945 raise errors.ConfigurationError("Configuration updated since object"
946 " has been read or unknown object")
947 target.serial_no += 1