[389-commits] [389-ds-base] branch 389-ds-base-1.4.3 updated: Issue 50545 - Port dbgen.pl to dsctl

This is an automated email from the git hooks/post-receive script.

mreynolds pushed a commit to branch 389-ds-base-1.4.3
in repository 389-ds-base.

The following commit(s) were added to refs/heads/389-ds-base-1.4.3 by this push:
new cdeaed9 Issue 50545 - Port dbgen.pl to dsctl
cdeaed9 is described below

commit cdeaed9000d77eafb4f04222e5b5fac5d86e4871
Author: Mark Reynolds <mreynolds@redhat.com>
AuthorDate: Thu Apr 9 17:32:37 2020 -0400

Issue 50545 - Port dbgen.pl to dsctl

Description: Ported the main features to lib389 and added some other useful features:

Now there are several LDIFs that can be created:

- User LDIFs (different types)
- Group LDIFs
- COS LDIFs
- Role LDIFs
- Modification LDIFs
- Nested LDIFs

Design Doc: https://www.port389.org/docs/389ds/design/dbgen-design.html

fixes: https://pagure.io/389-ds-base/issue/50545

Reviewed by: firstyear & spichugi(Thanks!!)

Fix various issue and improve ldif file validation

Add summary of settings to output, and set the default location of user/nested LDIF to be in the server's LDIF directory
---
dirsrvtests/tests/stress/search/simple.py | 12 +-
dirsrvtests/tests/suites/basic/basic_test.py | 7 +-
dirsrvtests/tests/suites/import/regression_test.py | 19 +-
.../mapping_tree/referral_during_tot_init_test.py | 16 +-
dirsrvtests/tests/suites/syntax/mr_test.py | 13 +-
src/lib389/cli/dsctl | 4 +-
src/lib389/lib389/cli_conf/chaining.py | 2 +-
src/lib389/lib389/cli_ctl/dbgen.py | 587 +++++++++++++++++++
src/lib389/lib389/cli_ctl/health.py | 5 +-
src/lib389/lib389/cli_ctl/nsstate.py | 4 +-
src/lib389/lib389/dbgen.py | 621 +++++++++++++++++++--
11 files changed, 1210 insertions(+), 80 deletions(-)

diff --git a/dirsrvtests/tests/stress/search/simple.py b/dirsrvtests/tests/stress/search/simple.py
index 8222cb7..d745ff4 100644
--- a/dirsrvtests/tests/stress/search/simple.py
+++ b/dirsrvtests/tests/stress/search/simple.py
@@ -1,5 +1,6 @@
# --- BEGIN COPYRIGHT BLOCK ---
# Copyright (C) 2019 William Brown <william@blackhats.net.au>
+# Copyright (C) 2020 Red Hat, Inc.
# All rights reserved.
#
# License: GPL (version 3 or any later version).
@@ -8,16 +9,15 @@
#

from lib389.topologies import topology_st
-from lib389.dbgen import dbgen
+from lib389.dbgen import dbgen_users
from lib389.ldclt import Ldclt
from lib389.tasks import ImportTask
-
from lib389._constants import DEFAULT_SUFFIX


def test_stress_search_simple(topology_st):
"""Test a simple stress test of searches on the directory server.
-
+
:id: 3786d01c-ea03-4655-a4f9-450693c75863
:setup: Standalone Instance
:steps:
@@ -31,7 +31,6 @@ def test_stress_search_simple(topology_st):
"""

inst = topology_st.standalone
-
inst.config.set("nsslapd-verify-filter-schema", "off")
# Bump idllimit to test OR worst cases.
from lib389.config import LDBMConfig
@@ -39,10 +38,9 @@ def test_stress_search_simple(topology_st):
# lconfig.set("nsslapd-idlistscanlimit", '20000')
# lconfig.set("nsslapd-lookthroughlimit", '20000')

-
ldif_dir = inst.get_ldif_dir()
import_ldif = ldif_dir + '/basic_import.ldif'
- dbgen(inst, 10000, import_ldif, DEFAULT_SUFFIX)
+ dbgen_users(inst, 10000, import_ldif, DEFAULT_SUFFIX)

r = ImportTask(inst)
r.import_suffix_from_ldif(ldiffile=import_ldif, suffix=DEFAULT_SUFFIX)
@@ -50,10 +48,8 @@ def test_stress_search_simple(topology_st):

# Run a small to warm up the server's caches ...
l = Ldclt(inst)
-
l.search_loadtest(DEFAULT_SUFFIX, "(mail=XXXX@example.com)", rounds=1)

# Now do it for realsies!
# l.search_loadtest(DEFAULT_SUFFIX, "(|(mail=XXXX@example.com)(nonexist=foo))", rounds=10)
l.search_loadtest(DEFAULT_SUFFIX, "(mail=XXXX@example.com)", rounds=10)
-
diff --git a/dirsrvtests/tests/suites/basic/basic_test.py b/dirsrvtests/tests/suites/basic/basic_test.py
index 40f95a4..1202073 100644
--- a/dirsrvtests/tests/suites/basic/basic_test.py
+++ b/dirsrvtests/tests/suites/basic/basic_test.py
@@ -1,5 +1,5 @@
# --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2016 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
# All rights reserved.
#
# License: GPL (version 3 or any later version).
@@ -18,10 +18,9 @@ import pytest
from lib389.tasks import *
from lib389.utils import *
from lib389.topologies import topology_st
-from lib389.dbgen import dbgen
+from lib389.dbgen import dbgen_users
from lib389.idm.organizationalunit import OrganizationalUnits
from lib389._constants import DN_DM, PASSWORD, PW_DM
-from lib389.topologies import topology_st
from lib389.paths import Paths
from lib389.idm.directorymanager import DirectoryManager
from lib389.config import LDBMConfig
@@ -266,7 +265,7 @@ def test_basic_import_export(topology_st, import_example_ldif):
log.info("Generating LDIF...")
ldif_dir = topology_st.standalone.get_ldif_dir()
import_ldif = ldif_dir + '/basic_import.ldif'
- dbgen(topology_st.standalone, 50000, import_ldif, DEFAULT_SUFFIX)
+ dbgen_users(topology_st.standalone, 50000, import_ldif, DEFAULT_SUFFIX)

# Online
log.info("Importing LDIF online...")
diff --git a/dirsrvtests/tests/suites/import/regression_test.py b/dirsrvtests/tests/suites/import/regression_test.py
index 7be9e39..e78d338 100644
--- a/dirsrvtests/tests/suites/import/regression_test.py
+++ b/dirsrvtests/tests/suites/import/regression_test.py
@@ -1,22 +1,23 @@
-# Copyright (C) 2017 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
# All rights reserved.
#
# License: GPL (version 3 or any later version).
# See LICENSE for details.
# --- END COPYRIGHT BLOCK ---
#
+import ldap
+import logging
+import os
import pytest
+import threading
+import time
from lib389.backend import Backends
from lib389.properties import TASK_WAIT
-from lib389.utils import time, ldap, os, logging
from lib389.topologies import topology_st as topo
-from lib389.dbgen import dbgen
+from lib389.dbgen import dbgen_users
from lib389._constants import DEFAULT_SUFFIX
from lib389.tasks import *
from lib389.idm.user import UserAccounts
-import threading
-import time
-
from lib389.idm.directorymanager import DirectoryManager

pytestmark = pytest.mark.tier1
@@ -145,7 +146,7 @@ def test_import_be_default(topo):
log.info('Create LDIF file and import it...')
ldif_dir = topo.standalone.get_ldif_dir()
ldif_file = os.path.join(ldif_dir, 'default.ldif')
- dbgen(topo.standalone, 5, ldif_file, TEST_DEFAULT_SUFFIX)
+ dbgen_users(topo.standalone, 5, ldif_file, TEST_DEFAULT_SUFFIX)

log.info('Stopping the server and running offline import...')
topo.standalone.stop()
@@ -185,7 +186,7 @@ def test_del_suffix_import(topo):
ldif_dir = topo.standalone.get_ldif_dir()
ldif_file = os.path.join(ldif_dir, 'suffix_del1.ldif')

- dbgen(topo.standalone, 10, ldif_file, TEST_SUFFIX1)
+ dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX1)

log.info('Stopping the server and running offline import')
topo.standalone.stop()
@@ -223,7 +224,7 @@ def test_del_suffix_backend(topo):
ldif_dir = topo.standalone.get_ldif_dir()
ldif_file = os.path.join(ldif_dir, 'suffix_del2.ldif')

- dbgen(topo.standalone, 10, ldif_file, TEST_SUFFIX2)
+ dbgen_users(topo.standalone, 10, ldif_file, TEST_SUFFIX2)

topo.standalone.tasks.importLDIF(suffix=TEST_SUFFIX2, input_file=ldif_file, args={TASK_WAIT: True})

diff --git a/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py b/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py
index 730969a..9f7e3e3 100644
--- a/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py
+++ b/dirsrvtests/tests/suites/mapping_tree/referral_during_tot_init_test.py
@@ -1,5 +1,5 @@
# --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2017 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
# All rights reserved.
#
# License: GPL (version 3 or any later version).
@@ -9,12 +9,10 @@
import ldap
import pytest
from lib389.topologies import topology_m2
-from lib389._constants import (DEFAULT_SUFFIX, HOST_MASTER_2, PORT_MASTER_2, TASK_WAIT)
+from lib389._constants import (DEFAULT_SUFFIX)
from lib389.agreement import Agreements
-
from lib389.idm.user import (TEST_USER_PROPERTIES, UserAccounts)
-
-from lib389.dbgen import dbgen
+from lib389.dbgen import dbgen_users
from lib389.utils import ds_is_older

pytestmark = pytest.mark.tier1
@@ -26,17 +24,15 @@ def test_referral_during_tot(topology_m2):
master2 = topology_m2.ms["master2"]

users = UserAccounts(master2, DEFAULT_SUFFIX)
-
u = users.create(properties=TEST_USER_PROPERTIES)
u.set('userPassword', 'password')
-
binddn = u.dn
bindpw = 'password'

# Create a bunch of entries on master1
ldif_dir = master1.get_ldif_dir()
import_ldif = ldif_dir + '/ref_during_tot_import.ldif'
- dbgen(master1, 10000, import_ldif, DEFAULT_SUFFIX)
+ dbgen_users(master1, 10000, import_ldif, DEFAULT_SUFFIX)

master1.stop()
master1.ldif2db(bename=None, excludeSuffixes=None, encrypt=False, suffixes=[DEFAULT_SUFFIX], import_file=import_ldif)
@@ -61,9 +57,7 @@ def test_referral_during_tot(topology_m2):
except ldap.REFERRAL:
referred = True
break
- # Means we never go a referral, should not happen!
+ # Means we never go a referral, should not happen!
assert referred

# Done.
-
-
diff --git a/dirsrvtests/tests/suites/syntax/mr_test.py b/dirsrvtests/tests/suites/syntax/mr_test.py
index f622b75..a7fa5a9 100644
--- a/dirsrvtests/tests/suites/syntax/mr_test.py
+++ b/dirsrvtests/tests/suites/syntax/mr_test.py
@@ -1,8 +1,17 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2020 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
import logging
import pytest
import os
import ldap
-from lib389.dbgen import dbgen
+from lib389.dbgen import dbgen_users
from lib389._constants import *
from lib389.topologies import topology_st as topo
from lib389._controls import SSSRequestControl
@@ -33,7 +42,7 @@ def test_sss_mr(topo):
log.info("Creating LDIF...")
ldif_dir = topo.standalone.get_ldif_dir()
ldif_file = os.path.join(ldif_dir, 'mr-crash.ldif')
- dbgen(topo.standalone, 5, ldif_file, DEFAULT_SUFFIX)
+ dbgen_users(topo.standalone, 5, ldif_file, DEFAULT_SUFFIX)

log.info("Importing LDIF...")
topo.standalone.stop()
diff --git a/src/lib389/cli/dsctl b/src/lib389/cli/dsctl
index f3bbabc..fd9bd87 100755
--- a/src/lib389/cli/dsctl
+++ b/src/lib389/cli/dsctl
@@ -12,7 +12,6 @@

import json
import argparse, argcomplete
-import logging
import sys
import signal
import os
@@ -23,9 +22,9 @@ from lib389.cli_ctl import dbtasks as cli_dbtasks
from lib389.cli_ctl import tls as cli_tls
from lib389.cli_ctl import health as cli_health
from lib389.cli_ctl import nsstate as cli_nsstate
+from lib389.cli_ctl import dbgen as cli_dbgen
from lib389.cli_ctl.instance import instance_remove_all
from lib389.cli_base import (
- _get_arg,
disconnect_instance,
setup_script_logger,
format_error_to_dict)
@@ -61,6 +60,7 @@ cli_dbtasks.create_parser(subparsers)
cli_tls.create_parser(subparsers)
cli_health.create_parser(subparsers)
cli_nsstate.create_parser(subparsers)
+cli_dbgen.create_parser(subparsers)

argcomplete.autocomplete(parser)

diff --git a/src/lib389/lib389/cli_conf/chaining.py b/src/lib389/lib389/cli_conf/chaining.py
index 5125898..380ec16 100644
--- a/src/lib389/lib389/cli_conf/chaining.py
+++ b/src/lib389/lib389/cli_conf/chaining.py
@@ -8,7 +8,7 @@

import json
from lib389.chaining import (
- ChainingLink, ChainingLinks, ChainingConfig, ChainingDefault)
+ ChainingLinks, ChainingConfig, ChainingDefault)
from lib389.cli_base import (
_generic_list,
_generic_get,
diff --git a/src/lib389/lib389/cli_ctl/dbgen.py b/src/lib389/lib389/cli_ctl/dbgen.py
new file mode 100644
index 0000000..7bc3892
--- /dev/null
+++ b/src/lib389/lib389/cli_ctl/dbgen.py
@@ -0,0 +1,587 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2020 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+
+from lib389.dbgen import (
+ dbgen_users,
+ dbgen_groups,
+ dbgen_cos_def,
+ dbgen_cos_template,
+ dbgen_role,
+ dbgen_mod_load,
+ dbgen_nested_ldif,
+)
+from lib389.utils import is_a_dn
+
+DEFAULT_LDIF = "/tmp/ldifgen.ldif"
+USERS_LDIF_NAME = "/users.ldif"
+ignore_args = [
+ "ldif_file",
+ "func",
+ "verbose",
+ "list",
+ "instance",
+ "json",
+ "remove_all",
+]
+
+def get_ldif_dir(instance):
+ """
+ Get the server's LDIF directory. This is only used for user & nested LDIFs
+ """
+ server_dir = instance.get_ldif_dir()
+ if server_dir is not None:
+ return server_dir
+ return DEFAULT_LDIF
+
+
+def adjust_ldif_name(instance, ldif_name):
+ """
+ If just a name is provided append it to the server's LDIF directory
+ """
+ if ldif_name[0] == '.' or ldif_name[0] == '/':
+ # Name appears to already be an absolute path
+ return ldif_name
+ else:
+ # Its just a name, add the server's ldif directory
+ return instance.get_ldif_dir() + "/" + ldif_name
+
+
+def display_args(log, args):
+ # Display all the options that are being used to generate the ldif file
+ log.info(f"\nGenerating LDIF with the following options:")
+ for k, v in vars(args).items():
+ if k in ignore_args or v is None:
+ continue
+ k = k.replace("_", "-") # Restore arg's original name
+ log.info(f" - {k}={v}")
+ log.info(f" - ldif-file={args.ldif_file}")
+ log.info("\nWriting LDIF ...")
+
+
+def validate_ldif_file(ldif_file, log=None):
+ """
+ Check if the LDIF file exists. If interactive then return some error
+ text to the caller, otherwise raise an error.
+ """
+ try:
+ f = open(ldif_file, 'w')
+ f.close()
+ return True
+ except PermissionError:
+ # File might already exist
+ msg = f"The LDIF file ({ldif_file}) exists and can not be overwritten. Please choose a different file name."
+ if log is not None:
+ log.info(msg)
+ else:
+ raise ValueError(msg)
+ except FileNotFoundError:
+ msg = f"The LDIF file ({ldif_file}) location does not exist. Please choose a different location."
+ if log is not None:
+ log.info(msg)
+ else:
+ raise ValueError(msg)
+ except Exception as e:
+ msg = f"The LDIF file ({ldif_file}) can not be written: {str(e)}"
+ if log is not None:
+ log.info(msg)
+ else:
+ raise ValueError(msg)
+
+ return False
+
+
+def get_ldif_file_input(log, default_name=DEFAULT_LDIF):
+ valid = False
+ while not valid:
+ file_name = get_input(log, "Enter the new LDIF file name", default_name)
+ valid = validate_ldif_file(file_name, log=log)
+ return file_name
+
+
+def get_input(log, msg, default, type="", options=None):
+ # Interactive prompt
+ display_default = default
+ if isinstance(default, bool):
+ if default:
+ display_default = "yes"
+ else:
+ display_default = "no"
+ while 1:
+ if display_default != "":
+ val = input(f'\n{msg} [{display_default}]: ')
+ else:
+ val = input(f'\n{msg}: ')
+ if val != '':
+ if type == "dn":
+ if is_a_dn(val, allow_anon=False):
+ return val
+ else:
+ log.info(f"\n ---> The value you entered \"{val}\" is not a valid DN")
+ continue
+ elif type == "int":
+ if val.isdigit():
+ if int(val) < 1:
+ log.info("\n ---> You must enter number greater than 0")
+ continue
+ return int(val)
+ else:
+ log.info(f"\n ---> The number you entered \"{val}\" is not a number")
+ continue
+ elif type == "bool":
+ if val.lower() == "y" or val.lower() == "yes":
+ return True
+ elif val.lower() == "n" or val.lower() == "no":
+ return False
+ else:
+ log.info(f"\n ---> Invalid value ({val}), please enter \"yes\" or \"no\".")
+ continue
+ else:
+ # Just a string, nothing to validate
+ if options is not None:
+ if val.lower() not in options:
+ opt_str = ', '.join(options)
+ log.info(f"\n ---> Invalid value ({val}), please enter one of the following types: {opt_str}")
+ continue
+ return val
+ else:
+ # User selected the default value
+ return default
+
+
+def dbgen_create_users(inst, log, args):
+ """
+ Create a LDIF of user entries
+ """
+ if args.number is None or args.suffix is None:
+ """
+ Interactively get all the info ...
+ """
+ log.info("Missing required parameters, switching to Interactive mode ...")
+
+ # Get the suffix
+ args.suffix = get_input(log, "Enter the suffix", "dc=example,dc=com", "dn")
+
+ # Get the parent
+ args.parent = get_input(log, "Enter the parent entry for the users", "ou=people,dc=example,dc=com", "dn")
+
+ # Get the number of users to create
+ args.number = get_input(log, "Enter the number of users to create", 100000, "int")
+
+ # Confirm the RDN attribute
+ args.rdn_cn = get_input(log, "Do you want to use \"cn\" instead of \"uid\" for the entry RDN attribute (yes/no)", False, "bool")
+
+ # Create generic entries
+ args.generic = get_input(log, "Create generic entries that can be used with \"ldclt\" (yes/no)", False, "bool")
+
+ # get offset size
+ if args.generic:
+ args.start_idx = get_input(log, "Choose the starting index for the generic user entries", 0, "int")
+
+ # localize the data
+ args.localize = get_input(log, "Do you want to localize the LDIF data (yes/no)", False, "bool")
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log, default_name=get_ldif_dir(inst) + USERS_LDIF_NAME)
+ else:
+ args.ldif_file = adjust_ldif_name(inst, args.ldif_file)
+ validate_ldif_file(args.ldif_file)
+
+ display_args(log, args)
+ dbgen_users(inst, args.number, args.ldif_file, args.suffix, generic=args.generic, parent=args.parent, startIdx=args.start_idx, rdnCN=False, pseudol10n=args.localize)
+ log.info(f"Successfully created LDIF file: {args.ldif_file}")
+
+
+def dbgen_create_groups(inst, log, args):
+ """
+ Create static groups and their members
+ """
+
+ if args.number is None or args.suffix is None:
+ """
+ Interactively get all the info ...
+ """
+ log.info("Missing required parameters, switching to Interactive mode ...")
+
+ # Get the number of users to create
+ args.number = get_input(log, "Enter the number of groups to create", 1, "int")
+
+ # Get the suffix
+ args.suffix = get_input(log, "Enter the suffix", "dc=example,dc=com", "dn")
+
+ # Get the parent
+ args.parent = get_input(log, "Enter the parent entry to add the groups under", args.suffix, "dn")
+
+ # Get the membership attr
+ args.member_attr = get_input(log, "Enter the attribute to use for the group membership", "uniquemember")
+
+ # Number of members
+ args.num_members = get_input(log, "Enter the number of members to add to the group", 10000, "int")
+
+ # Create member entries
+ args.create_members = get_input(log, "Do you want to create the member entries (yes/no)", True, "bool")
+
+ # member entries parent
+ if args.create_members:
+ args.member_parent = get_input(log, "Enter the parent entry to add the users under", args.suffix, "dn")
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log)
+ else:
+ validate_ldif_file(args.ldif_file)
+
+ props = {
+ "name": args.NAME,
+ "parent": args.parent,
+ "suffix": args.suffix,
+ "number": args.number,
+ "numMembers": args.num_members,
+ "createMembers": args.create_members,
+ "memberParent": args.member_parent,
+ "membershipAttr": args.member_attr,
+ }
+
+ display_args(log, args)
+ dbgen_groups(inst, args.ldif_file, props)
+ log.info(f"Successfully created LDIF file: {args.ldif_file}")
+
+
+def dbgen_create_cos_def(inst, log, args):
+ """
+ Create a COS definition
+ """
+ if args.type is None or args.parent is None or len(args.cos_attr) == 0 or \
+ ((args.type == "classic" or args.type == "indirect") and args.cos_specifier is None) \
+ or ((args.type == "classic" or args.type == "pointer") and args.cos_template is None):
+ log.info("Missing required parameters, switching to Interactive mode ...")
+
+ # Get the number of users to create
+ args.type = get_input(log, "Type of COS definition: \"classic\", \"pointer\", or \"indirect\"",
+ "classic", options=["classic", "pointer", "indirect"])
+
+ # Get the parent
+ args.parent = get_input(log, "Enter the parent entry to add the COS definition under", "dc=example,dc=com", "dn")
+
+ # Create parent
+ args.create_members = get_input(log, "Do you want to create the parent entry (yes/no)", True, "bool")
+
+ # COS specifier
+ if args.type == "classic" or args.type == "indirect":
+ args.cos_specifier = get_input(log, "Enter the COS specifier attribute", "description")
+
+ # COS template DN
+ if args.type == "classic" or args.type == "pointer":
+ args.cos_template = get_input(log, "Enter the COS Template DN", "cn=COS Template Entry,dc=example,dc=com", "dn")
+
+ # Gather the COS attributes
+ while True:
+ val = get_input(log, "Enter COS attributes, press Enter when finished", "")
+ if val == "" and len(args.cos_attr) > 0:
+ break
+ args.cos_attr.append(val)
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log)
+ else:
+ validate_ldif_file(args.ldif_file)
+
+ props = {
+ "cosType": args.type,
+ "defName": args.NAME,
+ "defParent": args.parent,
+ "defCreateParent": args.create_parent,
+ "cosSpecifier": args.cos_specifier,
+ "cosAttrs": args.cos_attr,
+ "tmpName": args.cos_template
+ }
+
+ display_args(log, args)
+ dbgen_cos_def(inst, args.ldif_file, props)
+ log.info(f"Successfully created LDIF file: {args.ldif_file}")
+
+
+def dbgen_create_cos_tmp(inst, log, args):
+ """
+ Create a COS template entry
+ """
+ if args.parent is None or args.cos_priority is None or args.cos_attr_val is None:
+ log.info("Missing required parameters, switching to Interactive mode ...")
+
+ # Get the parent
+ args.parent = get_input(log, "Enter the parent entry to add the COS template under", "dc=example,dc=com", "dn")
+
+ # Create parent
+ args.create_parent = get_input(log, "Do you want to create the parent entry (yes/no)", True, "bool")
+
+ # Get the COS priority
+ args.cos_priority = get_input(log, "Enter the COS priority for this template", "0", "int")
+
+ # Get the attribute value pair
+ args.cos_attr_val = get_input(log, "Enter the attribute and value pair. Use this format: \"ATTRIBUTE:VALUE\"", "postalcode:19605")
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log)
+ else:
+ validate_ldif_file(args.ldif_file)
+
+ props = {
+ "tmpName": args.NAME,
+ "tmpParent": args.parent,
+ "tmpCreateParent": args.create_parent,
+ "cosPriority": args.cos_priority,
+ "cosTmpAttrVal": args.cos_attr_val,
+ }
+
+ display_args(log, args)
+ dbgen_cos_template(inst, args.ldif_file, props)
+ log.info(f"Successfully created LDIF file: {args.ldif_file}")
+
+
+def dbgen_create_role(inst, log, args):
+ """
+ Create a Role
+ """
+ if args.type is None or args.parent is None or \
+ (args.type == "filtered" and args.filter is None) or \
+ (args.type == "nested" and len(args.role_dn) == 0):
+ log.info("Missing required parameters, switching to Interactive mode ...")
+
+ # Get the number of users to create
+ args.type = get_input(log, "Type of Role: \"managed\", \"filtered\", or \"nested\"",
+ "managed", options=["managed", "filtered", "nested"])
+
+ # Get the parent
+ args.parent = get_input(log, "Enter the parent entry to add the Role under", "dc=example,dc=com", "dn")
+
+ # Create parent
+ args.create_parent = get_input(log, "Do you want to create the parent entry (yes/no)", True, "bool")
+
+ # Role filter
+ if args.type == "filtered":
+ args.filter = get_input(log, "Enter the Role filter", "cn=some_value")
+
+ # Role DN (nested only)
+ if args.type == "nested":
+ while True:
+ val = get_input(log, "Enter the Role DN", "cn=some other role,dc=example,dc=com", "dn")
+ if val == "" and len(args.role_dn) > 0:
+ break
+ args.role_dn.append(val)
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log)
+ else:
+ validate_ldif_file(args.ldif_file)
+
+ props = {
+ "role_type": args.type,
+ "role_name": args.NAME,
+ "parent": args.parent,
+ "createParent": args.create_parent,
+ "filter": args.filter,
+ "role_list": args.role_dn
+ }
+
+ display_args(log, args)
+ dbgen_role(inst, args.ldif_file, props)
+ log.info(f"Successfully created LDIF file: {args.ldif_file}")
+
+
+def dbgen_create_mods(inst, log, args):
+ """
+ Create a LDIF file of update operations that can be consumed by ldapmodify
+
+ There are a lot of options here for creating different types of modification
+ LDIFs. One technique/option is that you can work with existing users. Use
+ dbgen to generate a large generic user ldif, import it, then you can create
+ a modification LDIF that will work on that database. It's a lot faster than
+ using ldapmodify to add 1 million first then modify those entries.
+
+ The other nice option is that you can randomize the operations, but this does
+ introduce a potential for some of the operations to fail. You could delete
+ an entry before you modify it, etc...
+ """
+
+ if args.num_users is None or args.parent is None:
+ log.info("Missing required parameters, switching to Interactive mode ...")
+
+ # Create users
+ args.create_users = get_input(log, "Do you want to create the user entries (yes/no)", True, "bool")
+
+ # Delete users
+ args.delete_users = get_input(log, "Do you want to delete all the user entries at the end (yes/no)", True, "bool")
+
+ # Get the number of entries
+ args.num_users = get_input(log, "Enter the number of user entries that can be modified", "100000", "int")
+
+ # Get the parent entry
+ args.parent = get_input(log, "The DN of the parent entry where the user entries are located", "ou=people,dc=example,dc=com")
+
+ # Create parent
+ args.create_parent = get_input(log, "Create the parent entry (yes/no)", True, "bool")
+
+ # Add users
+ args.add_users = get_input(log, "The number of users to add during the load", 0, "int")
+
+ # Delete users
+ args.del_users = get_input(log, "The number of users to delete during the load", 0, "int")
+
+ # modrdn users
+ args.modrdn_users = get_input(log, "The number of users to modrdn during the load", 0, "int")
+
+ # modify users
+ args.mod_users = get_input(log, "The number of users to modify during the load", 0, "int")
+
+ # Modification attributes
+ args.mod_attrs = get_input(log, "List of attributes that will be randomly chosen from when modifying an entry", "description cn")
+ args.mod_attrs = args.mod_attrs.split(' ')
+
+ # Randomize the load
+ args.randomize = get_input(log, "Randomly perform the specified add, mod, delete, and modrdn operations (yes/no)", True, "bool")
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log)
+ else:
+ validate_ldif_file(args.ldif_file)
+
+ props = {
+ "createUsers": args.create_users,
+ "deleteUsers": args.delete_users,
+ "numUsers": args.num_users,
+ "parent": args.parent,
+ "createParent": args.create_parent,
+ "addUsers": args.add_users,
+ "delUsers": args.del_users,
+ "modrdnUsers": args.modrdn_users,
+ "modUsers": args.mod_users,
+ "random": args.randomize,
+ "modAttrs": args.mod_attrs
+ }
+
+ display_args(log, args)
+ dbgen_mod_load(args.ldif_file, props)
+ log.info(f"Successfully created LDIF file: {args.ldif_file}")
+
+
+def dbgen_create_nested(inst, log, args):
+ """
+ Create a cascading/fractal tree. Every node splits in half and keeps
+ branching out in that fashion until all the entries are used up.
+ """
+
+ if args.num_users is None or args.node_limit is None or args.suffix is None:
+ # Num users
+ args.num_users = get_input(log, "The number of users to add during the load", 1000000, "int")
+
+ # Node limit
+ args.node_limit = get_input(log, "The total number of user entries to create under each node/subtree", 500, "int")
+
+ # Suffix
+ args.suffix = get_input(log, "Enter the suffix", "dc=example,dc=com", "dn")
+
+ # Get the output LDIF file name
+ args.ldif_file = get_ldif_file_input(log, default_name=get_ldif_dir(inst) + USERS_LDIF_NAME)
+ else:
+ args.ldif_file = adjust_ldif_name(inst, args.ldif_file)
+ validate_ldif_file(args.ldif_file)
+
+ props = {
+ "numUsers": int(args.num_users),
+ "nodeLimit": int(args.node_limit),
+ "suffix": args.suffix,
+ }
+
+ display_args(log, args)
+ node_count = dbgen_nested_ldif(inst, args.ldif_file, props)
+ log.info(f"Successfully created nested LDIF file ({args.ldif_file}) containing {node_count} nodes/subtrees")
+
+
+def create_parser(subparsers):
+ db_gen_parser = subparsers.add_parser('ldifgen', help="LDIF generator to make sample LDIF files for testing")
+ subcommands = db_gen_parser.add_subparsers(help="action")
+
+ # Create just users
+ dbgen_users_parser = subcommands.add_parser('users', help='Generate a LDIF containing user entries')
+ dbgen_users_parser.set_defaults(func=dbgen_create_users)
+ dbgen_users_parser.add_argument('--number', help="The number of users to create.")
+ dbgen_users_parser.add_argument('--suffix', help="The database suffix where the entries will be created.")
+ dbgen_users_parser.add_argument('--parent', help="The parent entry that the user entries should be created under. If not specified, the entries are stored under random Organizational Units.")
+ dbgen_users_parser.add_argument('--generic', action='store_true', help="Create generic entries in the format of \"uid=user####\". These entries are also compatible with ldclt.")
+ dbgen_users_parser.add_argument('--start-idx', default=0, help="For generic LDIF's you can choose the starting index for the user entries. The default is \"0\".")
+ dbgen_users_parser.add_argument('--rdn-cn', action='store_true', help="Use the attribute \"cn\" as the RDN attribute in the DN instead of \"uid\"")
+ dbgen_users_parser.add_argument('--localize', action='store_true', help="Localize the LDIF data")
+ dbgen_users_parser.add_argument('--ldif-file', default="users.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'users.ldif'")
+
+ # Create static groups
+ dbgen_groups_parser = subcommands.add_parser('groups', help='Generate a LDIF containing groups and members')
+ dbgen_groups_parser.set_defaults(func=dbgen_create_groups)
+ dbgen_groups_parser.add_argument('NAME', help="The group name.")
+ dbgen_groups_parser.add_argument('--number', default=1, help="The number of groups to create.")
+ dbgen_groups_parser.add_argument('--suffix', help="The database suffix where the groups will be created.")
+ dbgen_groups_parser.add_argument('--parent', help="The parent entry that the group entries should be created under. If not specified the groups are stored under the suffix.")
+ dbgen_groups_parser.add_argument('--num-members', default="10000", help="The number of members in the group. Default is 10000")
+ dbgen_groups_parser.add_argument('--create-members', action='store_true', help="Create the member user entries.")
+ dbgen_groups_parser.add_argument('--member-parent', help="The entry DN that the members should be created under. The default is the suffix entry.")
+ dbgen_groups_parser.add_argument('--member-attr', default="uniquemember", help="The membership attribute to use in the group. Default is \"uniquemember\".")
+ dbgen_groups_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"")
+
+ # Create a COS definition
+ dbgen_cos_def_parser = subcommands.add_parser('cos-def', help='Generate a LDIF containing a COS definition (classic, pointer, or indirect)')
+ dbgen_cos_def_parser.set_defaults(func=dbgen_create_cos_def)
+ dbgen_cos_def_parser.add_argument('NAME', help="The COS definition name.")
+ dbgen_cos_def_parser.add_argument('--type', help="The COS definition type: \"classic\", \"pointer\", or \"indirect\".")
+ dbgen_cos_def_parser.add_argument('--parent', help="The parent entry that the COS definition should be created under.")
+ dbgen_cos_def_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry")
+ dbgen_cos_def_parser.add_argument('--cos-specifier', help="Used in a classic COS definition, this attribute located in the user entry is used to select which COS template to use.")
+ dbgen_cos_def_parser.add_argument('--cos-template', help="The DN of the COS template entry, only used for \"classic\" and \"pointer\" COS definitions.")
+ dbgen_cos_def_parser.add_argument('--cos-attr', nargs='*', default=[], help="A list of attributes which defines which attribute the COS generates values for.")
+ dbgen_cos_def_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"")
+
+ # Create a COS Template
+ dbgen_cos_tmp_parser = subcommands.add_parser('cos-template', help='Generate a LDIF containing a COS template')
+ dbgen_cos_tmp_parser.set_defaults(func=dbgen_create_cos_tmp)
+ dbgen_cos_tmp_parser.add_argument('NAME', help="The COS template name.")
+ dbgen_cos_tmp_parser.add_argument('--parent', help="The DN of the entry to store the COS template entry under.")
+ dbgen_cos_tmp_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry")
+ dbgen_cos_tmp_parser.add_argument('--cos-priority', type=int, help="Sets the priority of this conflicting/competing COS templates.")
+ dbgen_cos_tmp_parser.add_argument('--cos-attr-val', help="defines the attribute and value that the template provides.")
+ dbgen_cos_tmp_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"")
+
+ # Create Role entries
+ dbgen_roles_parser = subcommands.add_parser('roles', help='Generate a LDIF containing a role entry (managed, filtered, or indirect)')
+ dbgen_roles_parser.set_defaults(func=dbgen_create_role)
+ dbgen_roles_parser.add_argument('NAME', help="The Role name.")
+ dbgen_roles_parser.add_argument('--type', help="The Role type: \"managed\", \"filtered\", or \"nested\".")
+ dbgen_roles_parser.add_argument('--parent', help="The DN of the entry to store the Role entry under")
+ dbgen_roles_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry")
+ dbgen_roles_parser.add_argument('--filter', help="A search filter for gathering Role members. Required for a \"filtered\" role.")
+ dbgen_roles_parser.add_argument('--role-dn', nargs='*', default=[], help="A DN of a role entry that should be included in this role. Used for \"nested\" roles only.")
+ dbgen_roles_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"")
+
+ # Create a modification LDIF
+ dbgen_mod_load_parser = subcommands.add_parser('mod-load', help='Generate a LDIF containing modify operations. This is intended to be consumed by ldapmodify.')
+ dbgen_mod_load_parser.set_defaults(func=dbgen_create_mods)
+ dbgen_mod_load_parser.add_argument('--create-users', action='store_true', help="Create the entries that will be modified or deleted. By default the script assumes the user entries already exist.")
+ dbgen_mod_load_parser.add_argument('--delete-users', action='store_true', help="Delete all the user entries at the end of the LDIF.")
+ dbgen_mod_load_parser.add_argument('--num-users', type=int, help="The number of user entries that will be modified or deleted")
+ dbgen_mod_load_parser.add_argument('--parent', help="The DN of the parent entry where the user entries are located.")
+ dbgen_mod_load_parser.add_argument('--create-parent', action='store_true', help="Create the parent entry")
+ dbgen_mod_load_parser.add_argument('--add-users', default=100, help="The number of additional entries to add during the load.")
+ dbgen_mod_load_parser.add_argument('--del-users', default=100, help="The number of entries to delete during the load.")
+ dbgen_mod_load_parser.add_argument('--modrdn-users', default=100, help="The number of entries to perform a modrdn operation on.")
+ dbgen_mod_load_parser.add_argument('--mod-users', default=100, help="The number of entries to modify.")
+ dbgen_mod_load_parser.add_argument('--mod-attrs', nargs="*", default=['description'], help="List of attributes the script will randomly choose from when modifying an entry. The default is \"description\".")
+ dbgen_mod_load_parser.add_argument('--randomize', action='store_true', help="Randomly perform the specified add, mod, delete, and modrdn operations")
+ dbgen_mod_load_parser.add_argument('--ldif-file', default=DEFAULT_LDIF, help=f"The LDIF file name. Default is \"{DEFAULT_LDIF}\"")
+
+ # Create a heavily nested LDIF
+ dbgen_nested_parser = subcommands.add_parser('nested', help='Generate a heavily nested database LDIF in a cascading/fractal tree design')
+ dbgen_nested_parser.set_defaults(func=dbgen_create_nested)
+ dbgen_nested_parser.add_argument('--num-users', help="The total number of user entries to create in the entire LDIF (does not include the container entries).")
+ dbgen_nested_parser.add_argument('--node-limit', help="The total number of user entries to create under each node/subtree")
+ dbgen_nested_parser.add_argument('--suffix', help="The suffix DN for the LDIF")
+ dbgen_nested_parser.add_argument('--ldif-file', default="nested-users.ldif", help=f"The LDIF file name. Default location is the server's LDIF directory using the name 'users.ldif'")
diff --git a/src/lib389/lib389/cli_ctl/health.py b/src/lib389/lib389/cli_ctl/health.py
index 10eaa44..3d15ad8 100644
--- a/src/lib389/lib389/cli_ctl/health.py
+++ b/src/lib389/lib389/cli_ctl/health.py
@@ -7,10 +7,9 @@
# --- END COPYRIGHT BLOCK ---

import json
-from getpass import getpass
-from lib389.cli_base import connect_instance, disconnect_instance, format_error_to_dict
+from lib389.cli_base import connect_instance, disconnect_instance
from lib389.cli_base.dsrc import dsrc_to_ldap, dsrc_arg_concat
-from lib389.backend import Backend, Backends
+from lib389.backend import Backends
from lib389.config import Encryption, Config
from lib389.monitor import MonitorDiskSpace
from lib389.replica import Replica, Changelog5
diff --git a/src/lib389/lib389/cli_ctl/nsstate.py b/src/lib389/lib389/cli_ctl/nsstate.py
index 20855a0..28acd99 100644
--- a/src/lib389/lib389/cli_ctl/nsstate.py
+++ b/src/lib389/lib389/cli_ctl/nsstate.py
@@ -42,7 +42,7 @@ def create_parser(subparsers):
repl_get_nsstate = subparsers.add_parser('get-nsstate', help="""Get the replication nsState in a human readable format

Replica DN: The DN of the replication configuration entry
-Replica SUffix: The replicated suffix
+Replica Suffix: The replicated suffix
Replica ID: The Replica identifier
Gen Time The time the CSN generator was created
Gen Time String: The time string of generator
@@ -61,4 +61,4 @@ Endian: Little/Big Endian
""")
repl_get_nsstate.add_argument('--suffix', default=False, help='The DN of the replication suffix to read the state from')
repl_get_nsstate.add_argument('--flip', default=False, help='Flip between Little/Big Endian, this might be required for certain architectures')
- repl_get_nsstate.set_defaults(func=get_nsstate)
+ repl_get_nsstate.set_defaults(func=get_nsstate)
diff --git a/src/lib389/lib389/dbgen.py b/src/lib389/lib389/dbgen.py
index f104ce8..6273781 100644
--- a/src/lib389/lib389/dbgen.py
+++ b/src/lib389/lib389/dbgen.py
@@ -1,5 +1,5 @@
# --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2017 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
# All rights reserved.
#
# License: GPL (version 3 or any later version).
@@ -8,12 +8,14 @@

# Replacement of the dbgen.pl utility

-from lib389.utils import pseudolocalize
+from lib389.utils import (ensure_str, pseudolocalize)
import random
import os
import pwd
import grp

+global node_count
+
DBGEN_POSITIONS = [
"Accountant",
"Admin",
@@ -66,21 +68,22 @@ DBGEN_LOCATIONS = [
]

DBGEN_OUS = [
-"Accounting",
-"Product Development",
-"Product Testing",
-"Human Resources",
-"Payroll",
-"People",
-"Groups",
+"accounting",
+"product development",
+"product testing",
+"human resources",
+"payroll",
+"people",
+"groups",
]

-DBGEN_TEMPLATE = """dn: {DN}
+DBGEN_TEMPLATE = """dn: {DN}{CHANGETYPE}
objectClass: top
objectClass: person
objectClass: organizationalPerson
objectClass: inetOrgPerson
-cn: {FIRST} {LAST}
+objectclass: inetUser
+cn: {CN}
sn: {LAST}
uid: {UID}
givenName: {FIRST}
@@ -100,7 +103,7 @@ l: {LOCATION}
ou: {OU}
mail: {UID}@example.com
mail: {UIDNUMBER}@example.com
-postalAddress: 518, Dept #851, Room#{OU}
+postalAddress: 518, Dept #851, Room#{OU}
title: {TITLE}
usercertificate;binary:: MIIBvjCCASegAwIBAgIBAjANBgkqhkiG9w0BAQQFADAnMQ8wDQYD
VQQDEwZjb25maWcxFDASBgNVBAMTC01NUiBDQSBDZXJ0MB4XDTAxMDQwNTE1NTEwNloXDTExMDcw
@@ -114,25 +117,109 @@ usercertificate;binary:: MIIBvjCCASegAwIBAgIBAjANBgkqhkiG9w0BAQQFADAnMQ8wDQYD

"""

-DBGEN_HEADER = """dn: {SUFFIX}
+DBGEN_OU_TEMPLATE = """dn: ou={OU},{SUFFIX}
objectClass: top
-objectClass: domain
-dc: {RDN}
-aci: (target=ldap:///{SUFFIX})(targetattr=*)(version 3.0; acl "acl1"; allow(write) userdn = "ldap:///self";)
-aci: (target=ldap:///{SUFFIX})(targetattr=*)(version 3.0; acl "acl2"; allow(write) groupdn = "ldap:///cn=Directory Administrators, {SUFFIX}";)
-aci: (target=ldap:///{SUFFIX})(targetattr=*)(version 3.0; acl "acl3"; allow(read, search, compare) userdn = "ldap:///anyone";)
+objectClass: organizationalUnit
+ou: {OU}

"""

-DBGEN_OU_TEMPLATE = """dn: ou={OU},{SUFFIX}
+RANDOM_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqurstuvwxyz0123456789_#@%&()?~$^`~*-=+{}|"\'.,<>'
+
+
+def finalize_ldif_file(instance, ldif_file):
+ # Make the file owned by dirsrv
+ os.chmod(ldif_file, 0o644)
+ if os.getuid() == 0:
+ # root user - chown the ldif to the server user
+ userid = ensure_str(instance.userid)
+ uid = pwd.getpwnam(userid).pw_uid
+ gid = grp.getgrnam(userid).gr_gid
+ os.chown(ldif_file, uid, gid)
+
+
+def get_index(idx, numUsers):
+ # Get ldclt style entry "0" padded index number
+ zeroLen = len(str(numUsers)) - len(str(idx))
+ index = '0' * zeroLen
+ index = index + str(idx)
+ return index
+
+
+def get_node(suffix):
+ # Build a node/container entry based on the suffix DN
+ rdn_attr = suffix.split('=')[0].lower()
+ rdn_attr_val = suffix.split('=')[1].split(',')[0]
+ if rdn_attr == 'c':
+ oc = 'country'
+ elif rdn_attr == 'cn':
+ oc = 'nscontainer'
+ elif rdn_attr == 'dc':
+ oc = 'domain'
+ elif rdn_attr == 'o':
+ oc = 'organization'
+ elif rdn_attr == 'ou':
+ oc = 'organizationalunit'
+ else:
+ # Unsupported rdn
+ raise ValueError("Suffix RDN '{}' in '{}' is not supported. Supported RDN's are: 'c', 'cn', 'dc', 'o', and 'ou'".format(rdn_attr, suffix))
+
+ return f"""dn: {suffix}
objectClass: top
-objectClass: organizationalUnit
-ou: {OU}
+objectClass: {oc}
+{rdn_attr}: {rdn_attr_val}
+aci: (target=ldap:///{suffix})(targetattr=*)(version 3.0; acl "Self Write"; allow(write) userdn = "ldap:///self";)
+aci: (target=ldap:///{suffix})(targetattr=*)(version 3.0; acl "Directory Admin Group"; allow(write) groupdn = "ldap:///cn=Directory Administrators,ou=Groups,{suffix}";)
+aci: (target=ldap:///{suffix})(targetattr=*)(version 3.0; acl "Anonymous Access"; allow(read, search, compare) userdn = "ldap:///anyone";)

"""


-def dbgen(instance, number, ldif_file, suffix, pseudol10n=False):
+def randomPick(values):
+ # Return a randomly selected value from the provided list of values
+ val_count = len(values)
+ val_count -= 1
+ idx = random.randint(0, val_count)
+ return values[idx].lstrip()
+
+
+def write_generic_user(LDIF, index, number, parent, name="user", changetype="", pseudol10n=False):
+ uid_val = name + get_index(index, number)
+ ou = random.choice(DBGEN_OUS)
+ first = uid_val
+ last = uid_val[::-1] # reverse name
+ cn = f"{first} {last}"
+ initials = "%s. %s" % (first[0], last[0])
+ l = random.choice(DBGEN_LOCATIONS)
+ title = "%s %s" % (random.choice(DBGEN_TITLE_LEVELS), random.choice(DBGEN_POSITIONS))
+ if pseudol10n:
+ ou = pseudolocalize(ou)
+ first = pseudolocalize(first)
+ last = pseudolocalize(last)
+ initials = pseudolocalize(initials)
+ l = pseudolocalize(l)
+ title = pseudolocalize(title)
+ dn = f"uid={uid_val},{parent}"
+
+ LDIF.write(DBGEN_TEMPLATE.format(
+ DN=dn,
+ CHANGETYPE=changetype,
+ UID=uid_val,
+ UIDNUMBER=index,
+ FIRST=first,
+ LAST=last,
+ CN=cn,
+ INITIALS=initials,
+ OU=ou,
+ LOCATION=l,
+ TITLE=title,
+ ))
+ return dn
+
+def dbgen_users(instance, number, ldif_file, suffix, generic=False, entry_name="user", parent=None, startIdx=0, rdnCN=False, pseudol10n=False):
+ """
+ Generate an LDIF of randomly named entries
+ """
familyname_file = os.path.join(instance.ds_paths.data_dir, 'dirsrv/data/dbgen-FamilyNames')
givename_file = os.path.join(instance.ds_paths.data_dir, 'dirsrv/data/dbgen-GivenNames')
familynames = []
@@ -142,20 +229,34 @@ def dbgen(instance, number, ldif_file, suffix, pseudol10n=False):
with open(givename_file, 'r') as f:
givennames = [n.strip() for n in f]

- with open(ldif_file, 'w') as output:
- rdn = suffix.split(",", 1)[0].split("=", 1)[1]
- output.write(DBGEN_HEADER.format(SUFFIX=suffix, RDN=rdn))
+ with open(ldif_file, 'w') as LDIF:
+ LDIF.write(get_node(suffix))
for ou in DBGEN_OUS:
ou = pseudolocalize(ou) if pseudol10n else ou
- output.write(DBGEN_OU_TEMPLATE.format(SUFFIX=suffix, OU=ou))
- for i in range(0, number):
+ LDIF.write(DBGEN_OU_TEMPLATE.format(SUFFIX=suffix, OU=ou))
+
+ if parent is not None:
+ parent_rdn = parent.split(',')[0].split('=')[1]
+ if parent_rdn.lower() not in DBGEN_OUS:
+ LDIF.write(get_node(parent))
+
+ for i in range(1, int(number) + 1):
# Pick a random ou
ou = random.choice(DBGEN_OUS)
first = random.choice(givennames)
last = random.choice(familynames)
+ if generic:
+ i += startIdx
+ name = entry_name + get_index(i, number)
+ uid = name
+ cn = name
+ else:
+ first = random.choice(givennames)
+ last = random.choice(familynames)
+ uid = "%s%s%s" % (first[0], last, i)
+ cn = f"{first} {last}"
# How do we subscript from a generator?
initials = "%s. %s" % (first[0], last[0])
- uid = "%s%s%s" % (first[0], last, i)
l = random.choice(DBGEN_LOCATIONS)
title = "%s %s" % (random.choice(DBGEN_TITLE_LEVELS), random.choice(DBGEN_POSITIONS))
if pseudol10n:
@@ -165,24 +266,468 @@ def dbgen(instance, number, ldif_file, suffix, pseudol10n=False):
initials = pseudolocalize(initials)
l = pseudolocalize(l)
title = pseudolocalize(title)
- dn = "uid=%s,ou=%s,%s" % (uid, ou, suffix)
- output.write(DBGEN_TEMPLATE.format(
+
+ if parent is None:
+ parent = f"ou={ou},{suffix}"
+
+ if rdnCN:
+ # Not using "uid" so use "cn" instead
+ dn = f"cn={cn},{parent}"
+ else:
+ dn = f"uid={uid},{parent}"
+
+ LDIF.write(DBGEN_TEMPLATE.format(
DN=dn,
+ CHANGETYPE="",
UID=uid,
UIDNUMBER=i,
FIRST=first,
LAST=last,
+ CN=cn,
INITIALS=initials,
OU=ou,
LOCATION=l,
TITLE=title,
- SUFFIX=suffix
))

- # Make the file owned by dirsrv
- os.chmod(ldif_file, 0o644)
- if os.getuid() == 0:
- # root user - chown the ldif to the server user
- uid = pwd.getpwnam(instance.userid).pw_uid
- gid = grp.getgrnam(instance.userid).gr_gid
- os.chown(ldif_file, uid, gid)
+ finalize_ldif_file(instance, ldif_file)
+
+
+def dbgen_groups(instance, ldif_file, props):
+ """
+ Create static group(s) and the member entries
+
+ props = {
+ "name": STRING,
+ "parent": DN,
+ "suffix": DN,
+ "number": ###, --> number of groups to create
+ "numMembers": ###,
+ "createMembers": True/False --> Create the member entries (default is True)
+ "memberParent": DN
+ "membershipAttr": ATTR
+ }
+ """
+ with open(ldif_file, 'w') as LDIF:
+ # Create the top node
+ LDIF.write(get_node(props['suffix']))
+ if props['parent'] is not None:
+ if props['parent'] != props['suffix']:
+ # Create the group container
+ LDIF.write(get_node(props['parent']))
+ else:
+ props['parent'] = props['suffix']
+
+ if props['memberParent'] is not None:
+ if props['memberParent'] != props['suffix'] and props['memberParent'] != props['parent']:
+ # Create the member/user container
+ LDIF.write(get_node(props['memberParent']))
+ else:
+ props['memberParent'] = props['suffix']
+
+ for idx in range(1, int(props['number']) + 1):
+ # Build the member list and create the member entries
+ group_member_list = []
+ if props['createMembers']:
+ for user_idx in range(1, int(props['numMembers']) + 1):
+ dn = write_generic_user(LDIF, user_idx, props['numMembers'], props['memberParent'], name=f"group_entry{idx}-")
+ group_member_list.append(dn)
+ else:
+ # Not creating the member entries, just build the DN list of members
+ for user_idx in range(1, int(props['numMembers']) + 1):
+ name = "user" + get_index(user_idx, props['numMembers'])
+ dn = f"uid={name},{props['memberParent']}"
+ group_member_list.append(dn)
+
+ if props['number'] == 0:
+ # Only creating one group, do not add the idx to DN
+ group_dn = f"dn: cn={props['name']},{props['parent']}\n"
+ cn = f"cn={props['name']},{props['parent']}"
+ else:
+ group_dn = f"dn: cn={props['name']}-{idx},{props['parent']}\n"
+ cn = f"cn={props['name']}-{idx},{props['parent']}"
+
+ LDIF.write(group_dn)
+ LDIF.write('objectclass: top\n')
+ LDIF.write('objectclass: groupOfUniqueNames\n')
+ LDIF.write('objectclass: groupOfNames\n')
+ LDIF.write('objectclass: inetAdmin\n')
+ LDIF.write(f'cn: {cn}\n')
+ for dn in group_member_list:
+ LDIF.write(f"{props['membershipAttr']}: {dn}\n")
+ LDIF.write('\n')
+
+ finalize_ldif_file(instance, ldif_file)
+
+
+def dbgen_cos_def(instance, ldif_file, props):
+ """
+ Create a COS definition
+
+ props = {
+ "cosType": "classic", "pointer", "indirect",
+ "defName": VAL,
+ "defParent": VAL,
+ "defCreateParent": True/False,
+ "cosSpecifier": can be used for cosIndirectSpecifier
+ "cosAttrs": [],
+ "tmpName": DN (need to classic and pointer COS defs)
+ }
+ """
+
+ if props['cosType'] == 'pointer':
+ objectclass = 'objectclass: cosPointerDefinition\n'
+ if props['cosType'] == 'indirect':
+ objectclass = 'objectclass: cosIndirectDefinition\n'
+ if props['cosType'] == 'classic':
+ objectclass = 'objectclass: cosClassicDefinition\n'
+
+ with open(ldif_file, 'w') as LDIF:
+ # Create parent
+ if props['defCreateParent']:
+ LDIF.write(get_node(props['defParent']))
+
+ #
+ # Create definition
+ #
+ dn = (f"dn: cn={props['defName']},{props['defParent']}\n")
+ LDIF.write(dn)
+ LDIF.write('objectclass: top\n')
+ LDIF.write('objectclass: cosSuperDefinition\n')
+ LDIF.write(objectclass)
+ LDIF.write('cn: ' + props['defName'] + "\n")
+
+ if props['cosType'] == 'pointer' or props['cosType'] == 'classic':
+ LDIF.write(f"cosTemplateDN: {props['tmpName']}\n")
+ if props['cosType'] == 'indirect':
+ LDIF.write(f"cosIndirectSpecifier: {props['cosSpecifier']}\n")
+ elif props['cosType'] == "classic":
+ LDIF.write(f"cosSpecifier: {props['cosSpecifier']}\n")
+ for attr in props['cosAttrs']:
+ # There can be multiple COS attributes
+ LDIF.write(f"cosAttribute: {attr}\n")
+
+ finalize_ldif_file(instance, ldif_file)
+
+
+def dbgen_cos_template(instance, ldif_file, props):
+ """
+ Create a COS Template
+
+ props = {
+ "tmpName": VAL,
+ "tmpParent": VAL,
+ "tmpCreateParent": True/False,
+ "cosPriority": ####
+ "cosTmpAttrVal": Attr/val
+ }
+ """
+
+ with open(ldif_file, 'w') as LDIF:
+ # Create parent
+ if props['tmpCreateParent']:
+ LDIF.write(get_node(props['tmpParent']))
+
+ # Create template
+ dn = f"dn: cn={props['tmpName']},{props['tmpParent']}\n"
+ LDIF.write(dn)
+ LDIF.write('objectclass: top\n')
+ LDIF.write('objectclass: extensibleObject\n')
+ LDIF.write('objectclass: cosTemplate\n')
+ LDIF.write(f"cn: {props['tmpName']}\n")
+ if props['cosPriority'] is not None:
+ LDIF.write(f"cosPriority: {props['cosPriority']}\n")
+ pair = props['cosTmpAttrVal'].split(':')
+ LDIF.write(f"{pair[0]}: {pair[1]}\n")
+
+ finalize_ldif_file(instance, ldif_file)
+
+
+def dbgen_role(instance, ldif_file, props):
+ """
+ Create a Role
+
+ props = {
+ role_type: "managed", "filter", pr "nested",
+ role_name: NAME,
+ parent: DN of parent,
+ createParent: True/False,
+ filter: FILTER,
+ role_list: [DN, DN, ...] # For nested role only
+ }
+ """
+
+ if props['role_type'].lower() == 'managed':
+ objectclasses = ('objectclass: nsSimpleRoleDefinition\n' +
+ 'objectclass: nsManagedRoleDefinition\n')
+ elif props['role_type'].lower() == 'filtered':
+ objectclasses = ('objectclass: nsComplexRoleDefinition\n' +
+ 'objectclass: nsFilteredRoleDefinition\n')
+ elif props['role_type'].lower() == 'nested':
+ objectclasses = ('objectclass: nsComplexRoleDefinition\n' +
+ 'objectclass: nsNestedRoleDefinition\n')
+
+ with open(ldif_file, 'w') as LDIF:
+ # Create parent entry
+ if props['createParent']:
+ LDIF.write(get_node(props['parent']))
+
+ dn = f"dn: cn={props['role_name']},{props['parent']}\n"
+ LDIF.write(dn)
+ LDIF.write('objectclass: top\n')
+ LDIF.write('objectclass: LdapSubEntry\n')
+ LDIF.write('objectclass: nsRoleDefinition\n')
+ LDIF.write(objectclasses)
+ LDIF.write(f"cn: {props['role_name']}\n")
+ if props['role_type'] == 'nested':
+ # Write out each role DN
+ for value in props['role_list']:
+ LDIF.write(f'nsRoleDN: {value}\n')
+ elif props['role_type'] == 'filtered':
+ LDIF.write(f"nsRoleFilter: {props['filter']}\n")
+
+ finalize_ldif_file(instance, ldif_file)
+
+
+def dbgen_mod_load(ldif_file, props):
+ """
+ Generate a "load" LDIF file that can be consumed by ldapmodify
+
+ props = {
+ "createUsers": True/False,
+ "deleteUsers": True/False,
+ "numUsers": ###,
+ "parent": DN, --> ou=people,dc=example,dc=com
+ "createParent": True/False,
+ "addUsers": ###,
+ "delUsers": ###,
+ 'modrdnUsers": ###,
+ "modUsers": ###, --> number of entries to modify
+ "random": True/False,
+ "modAttrs": [ATTR, ATTR, ...]
+ }
+ """
+
+ entry_dn_list = [] # List used to delete entries at the end of the LDIF
+ if props['modAttrs'] is None:
+ props['modAttrs'] = ['description', 'title']
+
+ with open(ldif_file, 'w') as LDIF:
+ if props['createParent']:
+ # Create the container entry that the users will be add to
+ LDIF.write(get_node(props['parent']))
+
+ # Create entries
+ for user_idx in range(1, props['numUsers'] + 1):
+ if props['createUsers']:
+ dn = write_generic_user(
+ LDIF, user_idx, props['numUsers'], props['parent'],
+ changetype="\nchangetype: add")
+ entry_dn_list.append(dn)
+ else:
+ dn = f"uid=user{get_index(user_idx, props['numUsers'])},{props['parent']}"
+ entry_dn_list.append(dn)
+
+ # Set the types of operations and how many of them to perform
+ addc = int(props['addUsers'])
+ delc = int(props['delUsers'])
+ modc = int(props['modUsers'])
+ mrdnc = int(props['modrdnUsers'])
+ total_ops = addc + delc + modc + mrdnc
+
+ if props['random']:
+ # Mix up the selected operations
+ operations = ['add', 'mod', 'modrdn', 'delete']
+ while total_ops != 0:
+ op = randomPick(operations)
+ if op == 'add':
+ if addc == 0:
+ # no more adds to do
+ operations.remove('add')
+ continue
+ dn = write_generic_user(
+ LDIF, addc, props['addUsers'], props['parent'],
+ name="addUser", changetype="\nchangetype: add")
+ entry_dn_list.append(dn)
+ addc -= 1
+ elif op == 'mod':
+ if modc == 0:
+ # no more mods to do
+ operations.remove('mod')
+ continue
+ attr = randomPick(props['modAttrs'])
+ val = (''.join((random.choice(RANDOM_CHARS) for i in range(0, random.randint(10, 30)))))
+ LDIF.write(f"dn: uid=user{get_index(modc, props['numUsers'])},{props['parent']}\n")
+ LDIF.write("changetype: modify\n")
+ LDIF.write(f"replace: {attr}\n")
+ LDIF.write(f"{attr}: {val}\n")
+ LDIF.write("\n")
+ modc -= 1
+ elif op == 'delete':
+ if delc == 0:
+ # no more deletes to do
+ operations.remove('delete')
+ continue
+
+ dn_val = f"uid=user{get_index(delc, props['numUsers'])},{props['parent']}"
+ LDIF.write(f"dn: {dn_val}\n")
+ LDIF.write("changetype: delete\n")
+ LDIF.write(" \n")
+ delc -= 1
+ if dn_val in entry_dn_list:
+ entry_dn_list.remove(dn_val)
+ elif op == 'modrdn':
+ if mrdnc == 0:
+ # no more modrdns to do
+ operations.remove('modrdn')
+ continue
+ dn_val = f"uid=user{get_index(mrdnc, props['numUsers'])},{props['parent']}"
+ new_dn_val = f"cn=user{get_index(mrdnc, props['numUsers'])},{props['parent']}"
+ LDIF.write(f"dn: {dn_val}\n")
+ LDIF.write("changetype: modrdn\n")
+ LDIF.write(f"newrdn: {new_dn_val}\n")
+ LDIF.write("deleteoldrdn: 1\n")
+ LDIF.write("\n")
+ mrdnc -= 1
+ # Revise the DN list: add the new DN, and remove the old DN
+ entry_dn_list.append(new_dn_val)
+ if dn_val in entry_dn_list:
+ entry_dn_list.remove(dn_val)
+
+ # Update the total count
+ total_ops -= 1
+ else:
+ # Sequentially do each type of operation
+
+ # Do the Adds
+ addc = int(props['addUsers'])
+ while addc != 0:
+ dn = write_generic_user(
+ LDIF, addc, props['addUsers'], props['parent'],
+ name="addUser", changetype="\nchangetype: add")
+ addc -= 1
+ entry_dn_list.append(dn)
+
+ # Mods
+ while modc != 0:
+ attr = randomPick(props['modAttrs'])
+ val = (''.join((random.choice(RANDOM_CHARS) for i in range(0, random.randint(10, 30)))))
+ LDIF.write(f"dn: uid=user{get_index(modc, props['numUsers'])},{props['parent']}\n")
+ LDIF.write("changetype: modify\n")
+ LDIF.write(f"replace: {attr}\n")
+ LDIF.write(f"{attr}: {val}\n")
+ LDIF.write("\n")
+ modc -= 1
+
+ # Modrdns
+ while mrdnc != 0:
+ dn_val = f"uid=user{get_index(mrdnc, props['numUsers'])},{props['parent']}"
+ new_dn_val = f"cn=user{get_index(mrdnc, props['numUsers'])},{props['parent']}"
+ LDIF.write(f"dn: {dn_val}\n")
+ LDIF.write("changetype: modrdn\n")
+ LDIF.write(f"newrdn: {new_dn_val}\n")
+ LDIF.write("deleteoldrdn: 1\n")
+ LDIF.write("\n")
+ mrdnc -= 1
+ # Revise the DN list: add the new DN, and remove the old DN
+ entry_dn_list.append(new_dn_val)
+ if dn_val in entry_dn_list:
+ entry_dn_list.remove(dn_val)
+
+ # Deletes
+ while delc != 0:
+ dn_val = f"uid=user{get_index(delc, props['numUsers'])},{props['parent']}"
+ LDIF.write(f"dn: {dn_val}\n")
+ LDIF.write("changetype: delete\n")
+ LDIF.write(" \n")
+ delc -= 1
+ if dn_val in entry_dn_list:
+ entry_dn_list.remove(dn_val)
+
+ # Cleanup - delete all known entries
+ if props['deleteUsers']:
+ for dn in entry_dn_list:
+ LDIF.write(f"dn: {dn}\n")
+ LDIF.write("changetype: delete\n")
+ LDIF.write(" \n")
+
+
+def build_recursive_nodes(LDIF, dn, node_limit, max_entries):
+ """
+ Recursively create two nodes under each node, continue until there are no
+ more max_entries.
+ """
+ global node_count
+ wrote_node1 = False
+ wrote_node2 = False
+
+ # Create containers for DN1 and DN2
+ dn1 = "ou=1," + dn
+ LDIF.write(f'dn: {dn1}\n')
+ LDIF.write('objectclass: top\n')
+ LDIF.write('objectclass: organizationalUnit\n')
+ LDIF.write('ou: ou=1\n\n')
+
+ dn2 = "ou=2," + dn
+ LDIF.write(f'dn: {dn2}\n')
+ LDIF.write('objectclass: top\n')
+ LDIF.write('objectclass: organizationalUnit\n')
+ LDIF.write('ou: ou=2\n\n')
+
+ # Add entries under each node
+ for entry_idx in range(1, node_limit + 1):
+ write_generic_user(LDIF, entry_idx, node_limit, dn1)
+ max_entries -= 1
+ if not wrote_node1:
+ wrote_node1 = True
+ node_count += 1
+ if max_entries == 0:
+ break
+
+ write_generic_user(LDIF, entry_idx, node_limit, dn2)
+ max_entries -= 1
+ if not wrote_node2:
+ wrote_node2 = True
+ node_count += 1
+ if max_entries == 0:
+ break
+
+ # Are we out of entries
+ if max_entries == 0:
+ return
+
+ # Get the remaining entries to be split between two nodes
+ new_node_max = max_entries // 2
+ if (max_entries % 2) > 0:
+ remainder = 1
+ else:
+ remainder = 0
+
+ # Recursively build child nodes
+ build_recursive_nodes(LDIF, dn1, node_limit, new_node_max)
+ build_recursive_nodes(LDIF, dn2, node_limit, new_node_max + remainder)
+
+
+def dbgen_nested_ldif(instance, ldif_file, props):
+ """
+ Create a deeply nested LDIF
+
+ props = {
+ "numUsers": #### --> Total number of user entries to create
+ 'nodeLimit': #### --> max number of entries to put into each node
+ "suffix": DN
+ }
+ """
+
+ global node_count
+ node_count = 0
+ with open(ldif_file, 'w') as LDIF:
+ # Create the top suffix
+ LDIF.write(get_node(props['suffix']))
+
+ # Create all the nodes
+ build_recursive_nodes(LDIF, props['suffix'], props['nodeLimit'], props['numUsers'])
+
+ finalize_ldif_file(instance, ldif_file)
+
+ return node_count

--
To stop receiving notification emails like this one, please contact
the administrator of this repository.
_______________________________________________
389-commits mailing list -- 389-commits@lists.fedoraproject.org
To unsubscribe send an email to 389-commits-leave@lists.fedoraproject.org
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedoraproject.org/archives/list/389-commits@lists.fedoraproject.org