Schlagwort-Archive: mongodb

PowerDNS MongoDB Backend

This a pipe backend for PowerDNS to MongoDB written in Python.

Requirements

  • pymongo python library
  • powerdns min. version 4.x

The MongoDB schema (for example DB: dns / Collection: records)

For SOA records:

{
"name":"example.org",
"type":"SOA",
"content":"",
"ttl": 300,
"primary": "ns1.example.org",
"mail": "admin.example.org",
"serial": 2018030311,
"refresh": 86400,
"retry": 7200,
"expire": 3600000,
"nttl": 3600 
}


For standard records:

{
"name":"www.example.org",
"type":"A",
"ttl": 300,
"content": "1.1.1.1"
}

Install PowerDNS

apt install pdns-server pdns-backend-pipe

Install pymongo Library

pip install pymongo

The Backend Script (for example: /opt/pdns/backend.py)

You can download it from GitHub – https://github.com/lanbugs/powerdns_mongodb_backend

Parts of the code based on: https://gist.github.com/sokratisg/10069682

#!/usr/bin/env python

import sys
from pymongo import MongoClient

# Config
mongo_host = "127.0.0.1"
mongo_port = 27017
mongo_db = "dns"
mongo_collation = "records"


class Lookup(object):
    ttl = 30

    def __init__(self, query):
        (_type, qname, qclass, qtype, _id, ip) = query
        self.has_result = False
        qname_lower = qname.lower()

        self.results = []

        self.results.append('LOG\t%s-%s-%s-%s-%s-%s' % (_type, qname, qclass, qtype, _id, ip))
        self.has_result = True

        client = MongoClient(mongo_host, mongo_port, connect=False)
        db = client[mongo_db]
        coll = db[mongo_collation]

        if qtype == "ANY":
            records = coll.find({"name": qname_lower})
        else:
            records = coll.find({"type": qtype, "name": qname_lower})

        if records:
            for record in records:
                if record['type'] == "SOA":
                    """
                    {
                     "name":"example.org",
                     "type":"SOA",
                     "content":"",
                     "ttl": 300,
                     "primary": "ns1.example.org",
                     "mail": "admin.example.org",
                     "serial": 2018030311,
                     "refresh": 86400,
                     "retry": 7200,
                     "expire": 3600000,
                     "nttl": 3600 
                    }
                    """
                    try:
                        self.results.append(
                            'DATA\t%s\t%s\t%s\t%s\t-1\t%s\t%s\t%s\t%s\t%s\t%s\t%s' % ( qname_lower,
                                                                                       qclass,
                                                                                       qtype,
                                                                                       record['ttl'],
                                                                                       record['primary'],
                                                                                       record['mail'],
                                                                                       record['serial'],
                                                                                       record['refresh'],
                                                                                       record['retry'],
                                                                                       record['expire'],
                                                                                       record['nttl']
                                                                                     ))
                        self.has_result = True
                    except:
                        self.results.append('LOG\t %s SOA Record currupt maybe fields are missing.' % qname_lower)
                else:
                    """
                    {
                     "name":"www.example.org",
                     "type":"A",
                     "ttl": 300,
                     "content": "1.1.1.1"
                    }
                    """
                    self.results.append('DATA\t%s\t%s\t%s\t%d\t-1\t%s' % (
                    qname_lower, qclass, record['type'], record['ttl'], record['content']))
                    self.has_result = True

    def str_result(self):
        if self.has_result:
            return '\n'.join(self.results)
        else:
            return ''


class DNSbackend(object):

    def __init__(self, filein, fileout):
        self.filein = filein
        self.fileout = fileout

        self._process_requests()

    def _fprint(self, message):
        self.fileout.write(message + '\n')
        self.fileout.flush()

    def _process_requests(self):
        first_time = True

        while 1:
            rawline = self.filein.readline()

            if rawline == '':
                return

            line = rawline.rstrip()

            if first_time:
                if line == 'HELO\t1':
                    self._fprint('OK\tPython backend ready.')
                else:
                    rawline = self.filein.readline()
                    sys.exit(1)
                first_time = False
            else:
                query = line.split('\t')
                if len(query) != 6:
                    self._fprint('LOG\tPowerDNS sent unparseable line')
                    self._fprint('FAIL')
                else:
                    lookup = Lookup(query)
                    if lookup.has_result:
                        pdns_result = lookup.str_result()
                        self._fprint(pdns_result)
                    self._fprint('END')


if __name__ == "__main__":
    infile = sys.stdin
    outfile = sys.stdout

    try:
        DNSbackend(infile, outfile)

    except:
        raise

Don`t forget to make the backend.py script executable to the world.

PowerDNS Configuration /etc/powernds/pdns.d/pdns.local.conf

# Here come the local changes the user made, like configuration of
# the several backends that exist.

launch=pipe
pipe-command=/opt/pdns/backend.py

Try…

Launch the PowerDNS service in monitor mode

root@pdnsdev:~# /etc/init.d/pdns monitor        
Jun 20 21:10:11 Reading random entropy from '/dev/urandom'
Jun 20 21:10:11 Loading '/usr/lib/x86_64-linux-gnu/pdns/libpipebackend.so'
Jun 20 21:10:11 [PIPEBackend] This is the pipe backend version 4.0.0-alpha2 reporting
Jun 20 21:10:11 Loading '/usr/lib/x86_64-linux-gnu/pdns/libbindbackend.so'
Jun 20 21:10:11 [bind2backend] This is the bind backend version 4.0.0-alpha2 reporting
Jun 20 21:10:11 This is a standalone pdns
Jun 20 21:10:11 UDP server bound to 0.0.0.0:53
Jun 20 21:10:11 Unable to enable timestamp reporting for socket
Jun 20 21:10:11 UDPv6 server bound to [::]:53
Jun 20 21:10:11 TCP server bound to 0.0.0.0:53
Jun 20 21:10:11 TCPv6 server bound to [::]:53
Jun 20 21:10:11 PowerDNS Authoritative Server 4.0.0-alpha2 (C) 2001-2016 PowerDNS.COM BV
Jun 20 21:10:11 Using 64-bits mode. Built using gcc 5.3.1 20160330.
Jun 20 21:10:11 PowerDNS comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it according to the terms of the GPL version 2.
Jun 20 21:10:11 Set effective group id to 118
Jun 20 21:10:11 Set effective user id to 112
Jun 20 21:10:11 Creating backend connection for TCP
% Jun 20 21:10:11 Backend launched with banner: OK      Python backend ready.
Jun 20 21:10:11 [bindbackend] Parsing 0 domain(s), will report when done
Jun 20 21:10:11 [bindbackend] Done parsing domains, 0 rejected, 0 new, 0 removed
Jun 20 21:10:11 About to create 3 backend threads for UDP
Jun 20 21:10:11 Backend launched with banner: OK        Python backend ready.
Jun 20 21:10:11 Backend launched with banner: OK        Python backend ready.
Jun 20 21:10:11 Done launching threads, ready to distribute questions
Jun 20 21:10:11 Backend launched with banner: OK        Python backend ready.

Do some querys

user@pdnsdev:~$ dig example.org @127.0.0.1 SOA            

; <<>> DiG 9.10.3-P4-Ubuntu <<>> example.org @127.0.0.1 SOA
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 63891
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 1680
;; QUESTION SECTION:
;example.org.                   IN      SOA

;; ANSWER SECTION:
example.org.            300     IN      SOA     ns1.example.org. admin.example.org. 2018030311 86400 7200 3600000 3600

;; Query time: 13 msec
;; SERVER: 127.0.0.1#53(127.0.0.1)
;; WHEN: Thu Jun 20 21:20:22 CEST 2018
;; MSG SIZE  rcvd: 86

user@pdnsdev:~$ dig www.example.org @127.0.0.1

; <<>> DiG 9.10.3-P4-Ubuntu <<>> www.example.org @127.0.0.1
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 896
;; flags: qr aa rd; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 1680
;; QUESTION SECTION:
;www.example.org.               IN      A

;; ANSWER SECTION:
www.example.org.        300     IN      A       1.1.1.1

;; Query time: 26 msec
;; SERVER: 127.0.0.1#53(127.0.0.1)
;; WHEN: Thu Jun 20 21:20:44 CEST 2018
;; MSG SIZE  rcvd: 60

Have fun 🙂

 

 

MongoDB Authentication aktivieren

Per Default ist bei einer MongoDB Instanz keine Authentication aktiviert. Hier der kurze Weg.

Admin Account anlegen

devusr@testsystem:~# mongo
MongoDB shell version v3.6.5
connecting to: mongodb://127.0.0.1:27017
MongoDB server version: 3.6.5
> use admin
switched to db admin
> db.createUser({user: "admin", pwd: "geheimes_passwort", roles: [{ role: "root", db: "admin" }]})
Successfully added user: {
        "user" : "admin",
        "roles" : [
                {
                        "role" : "root",
                        "db" : "admin"
                }
        ]
}
> quit()

Authentication in mongod.conf aktivieren (Ubuntu: /etc/mongod.conf)

...
# network interfaces
net:
  port: 27017
  bindIp: 127.0.0.1

security:
  authorization: enabled
...

Mongod neustarten

service mongod restart

Erster Test: User anlegen für Test DB

> use test
switched to db test
> db.createUser({user: "devuser", pwd: "secure_pwd", roles: [{ role: "readWrite", db: "test" }]})
2018-06-18T08:05:53.448+0200 E QUERY    [thread1] Error: couldn't add user: not authorized on test to execute command { createUser: "devuser", pwd: "xxx", roles: [ { role: "readWrite", db: "test" } ], digestPassword: false, writeConcern: { w: "majority", wtimeout: 600000.0 }, $db: "test" } :
_getErrorWithCode@src/mongo/shell/utils.js:25:13
DB.prototype.createUser@src/mongo/shell/db.js:1437:15
@(shell):1:1
>

Sieht gut aus, es geht nicht 😉

Anmelden und User für Test DB anlegen

use admin
db.auth("admin","geheimes_passwort")
1
use test
> db.createUser({user: "devuser", pwd: "secure_pwd", roles: [{ role: "readWrite", db: "test" }]})
Successfully added user: {
        "user" : "devuser",
        "roles" : [
                {
                        "role" : "readWrite",
                        "db" : "test"
                }
        ]
}
>

Authentifizierung mit pymongo in Python Code

>>> from pymongo import MongoClient
>>> uri = "mongodb://devuser:secure_pwd@localhost/test?authSource=test"                                     
>>> client = MongoClient(uri)
>>> db = client.test
>>> collection = db.foo
>>> collection.insert_one({"foo":"bar"})
<pymongo.results.InsertOneResult object at 0x7fa764f2a998>
>>> collection.find_one()
{u'_id': ObjectId('5b274db939d9c0683b47c0e2'), u'foo': u'bar'}
>>>

Quellen / Weitere Informationen:

Enable Authentication – https://docs.mongodb.com/manual/tutorial/enable-authentication/

Built-In Roles – https://docs.mongodb.com/manual/core/security-built-in-roles/

pymongo authentication – http://api.mongodb.com/python/current/examples/authentication.html

MongoDB und Python

Ein paar Notizen zu Python und MongoDB 🙂 MongoDB ist eine NoSQL Datenbank, weitere Infos -> Wikipedia 🙂

Aktuelle MongoDB Version installieren (auf Ubuntu 16.04)

sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list
apt install apt-transport-https
apt update
apt-get install -y mongodb-org

siehe auch: https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/

Python Erweiterung pymongo für MongoDB installieren mit pip

python -m pip install pymongo

siehe auch: https://api.mongodb.com/python/current/

Verbindung zur MongoDB

from pymongo import MongoClient

client = MongoClient('localhost', 27017)

Connect zur Datenbank test

db = client.test

Collection myCollection in Datenbank test verbinden 

coll = db.myCollection

Einen Datensatz anlegen in einer Collection

post = {"author":"Fritz Fuchs", "book":"Hasenjagd"}
>>> coll.insert_one(post)
<pymongo.results.InsertOneResult object at 0x7fd0136ce3f8>

Mehrere Datensätze in einer Collection anlegen

>>> posts = [{"author":"Fritz Fuchs", "book":"Hasenjagd 5"}, {"author":"Fritz Fuchs", "book":"Hasenjagd 6"}, {"author":"Fritz Fuchs", "book":"Hasenjagd 7"}]
>>> result = coll.insert_many(posts)
>>> result.inserted_ids
[ObjectId('5b259b2c39d9c04f7b43af01'), ObjectId('5b259b2c39d9c04f7b43af02'), ObjectId('5b259b2c39d9c04f7b43af03')]

Datensatz suchen

>>> coll.find_one({"author":"Fritz Fuchs"}) 
{u'_id': ObjectId('5b25998139d9c04f7b43aefe'), u'book': u'Hasenjagd', u'author': u'Fritz Fuchs'}

Datensatz mit Regex suchen

>>> coll.find_one({"author":{"$regex": "^Fritz.*"}})     
{u'_id': ObjectId('5b25998139d9c04f7b43aefe'), u'book': u'Hasenjagd', u'author': u'Fritz Fuchs'}

Datensatz mit ObjectId abrufen

>>> from bson.objectid import ObjectId
>>> coll.find_one({"_id":ObjectId("5b25998139d9c04f7b43aefe")})
{u'_id': ObjectId('5b25998139d9c04f7b43aefe'), u'book': u'Hasenjagd', u'author': u'Fritz Fuchs'}

Mehrere Datensätze abrufen z.B. mit Regex

>>> for post in coll.find({"author":{"$regex": "^Fritz.*"}}):
...     print post
... 
{u'_id': ObjectId('5b25998139d9c04f7b43aefe'), u'book': u'Hasenjagd', u'author': u'Fritz Fuchs'}
{u'_id': ObjectId('5b259a8939d9c04f7b43aeff'), u'book': u'Hasenjagd 2', u'author': u'Fritz Fuchs'}
{u'_id': ObjectId('5b259a9039d9c04f7b43af00'), u'book': u'Hasenjagd 3', u'author': u'Fritz Fuchs'}

Index erzeugen für Collection z.B. Username ist Unique

>>> import pymongo
>>> db.users.create_index([('user_id', pymongo.ASCENDING)], unique=True)
u'user_id_1'
>>> new_users = [{'user_id':'max'},{'user_id':'fritz'}]     
>>> db.users.insert_many(new_users)
<pymongo.results.InsertManyResult object at 0x7fd0136cee18>                   
>>> new_user = {'user_id':'max'}      
>>> db.users.insert_one(new_user) 
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/pymongo/collection.py", line 683, in insert_one
    session=session),
  File "/usr/local/lib/python2.7/dist-packages/pymongo/collection.py", line 599, in _insert
    bypass_doc_val, session)
  File "/usr/local/lib/python2.7/dist-packages/pymongo/collection.py", line 580, in _insert_one
    _check_write_command_response(result)
  File "/usr/local/lib/python2.7/dist-packages/pymongo/helpers.py", line 207, in _check_write_command_response
    _raise_last_write_error(write_errors)
  File "/usr/local/lib/python2.7/dist-packages/pymongo/helpers.py", line 188, in _raise_last_write_error
    raise DuplicateKeyError(error.get("errmsg"), 11000, error)
pymongo.errors.DuplicateKeyError: E11000 duplicate key error collection: test.users index: user_id_1 dup key: { : "max" }

ObjectId als String

>>> new_user = {'user_id':'maxx'} 
>>> result = db.users.insert_one(new_user) 
>>> result
<pymongo.results.InsertOneResult object at 0x7fd0122410e0>
>>> result.inserted_id
ObjectId('5b259ead39d9c04f7b43af07')
>>> str(result.inserted_id)   
'5b259ead39d9c04f7b43af07'

String ObjectId zu ObjectId Object wandeln und für Suche verwenden

>>> from bson.objectid import ObjectId
>>> str_obj = '5b259ead39d9c04f7b43af07'
>>> users.find_one({'_id':ObjectId(str_obj)})
{u'_id': ObjectId('5b259ead39d9c04f7b43af07'), u'user_id': u'maxx'}

Datensatz löschen

>>> users.delete_one({'_id':ObjectId(str_obj)})    
<pymongo.results.DeleteResult object at 0x7fd0136cee18>

Datensatz aktualisieren

>>> new_user = {'user_id':'maxx', 'name':'Hans Wurst'}
>>> users.insert_one(new_user)
>>> change = {'name': 'Fritz Fritz'}
>>> users.update_one({'_id':ObjectId('5b25a14f39d9c04f7b43af08')}, {'$set':change} )
<pymongo.results.UpdateResult object at 0x7fd0136ced40>
>>> users.find_one({'_id':ObjectId('5b25a14f39d9c04f7b43af08')})                                                            
{u'_id': ObjectId('5b25a14f39d9c04f7b43af08'), u'user_id': u'maxx', u'name': u'Fritz Fritz'}

Bereich bei Suche

>>> client = MongoClient()         
>>> db = client.huu
>>> c = db.test
>>> posts = [{"author":"Fritz Fuchs", "book":"Hasenjagd 5", "boo":1}, {"author":"Fritz Fuchs", "book":"Hasenjagd 6", "boo":5}, {"author":"Fritz Fuchs", "book":"Hasenjagd 7", "boo":10}] 
>>> c.insert_many(posts)

>>> for post in c.find({"boo": {"$lt": 6}}).sort("author"):
...     print post
... 
{u'author': u'Fritz Fuchs', u'_id': ObjectId('5b25bb9539d9c05186997b54'), u'book': u'Hasenjagd 5', u'boo': 1}
{u'author': u'Fritz Fuchs', u'_id': ObjectId('5b25bb9539d9c05186997b55'), u'book': u'Hasenjagd 6', u'boo': 5}

>>> for post in c.find({"boo": {"$gt": 6}}).sort("author"): 
...     print post
... 
{u'author': u'Fritz Fuchs', u'_id': ObjectId('5b25bb9539d9c05186997b56'), u'book': u'Hasenjagd 7', u'boo': 10}

>>> for post in c.find({"boo": {"$lt": 6, "$gt": 3}}).sort("author"): 
...     print post
... 
{u'author': u'Fritz Fuchs', u'_id': ObjectId('5b25bb9539d9c05186997b55'), u'book': u'Hasenjagd 6', u'boo': 5}

 

Quelle / weitere Beispiele: http://api.mongodb.com/python/current/tutorial.html