Planet PostgreSQL

(note: this series of articles is my personal opinion as a PostgreSQL core team member) · About

[barman]
... // General settings
retention_policy: REDUNDANCY 4
[malcolm]
description = Malcolm Rocks
ssh_command = ssh malcolm
conninfo = host=malcolm port=5432 user=postgres dbname=postgres
retention_policy: REDUNDANCY 8
[angus]
description = Angus Rocks
ssh_command = ssh angus
conninfo = host=angus port=5432 user=postgres dbname=postgres
retention_policy: RECOVERY WINDOW OF 3 MONTHS
initdb [OPTION]... [DATADIR]
   ........
   -k, --data-checksums      use data page checksums

initdb -D data_directory -k 

postgres=# select * from corruption_test;
WARNING:  page verification failed, calculated checksum 63023 but expected 48009
ERROR:  invalid page in block 0 of relation base/12896/16409

postgres=# select * from corruption_test where id=1;
ERROR:  invalid page header in block 0 of relation base/12870/18192

$ export PGDATA=/usr/local/pg93beta/data
$ pg_controldata 
....
....
....
Data page checksum version:           1

CREATE FOREIGN TABLE hugeset (value text)
SERVER localredis
OPTIONS (tabletype 'list', singleton_key 'myhugelist');

9,Karnataka,कर्नाटक
10,Kerala,केरळा

create table states(state_code int, state_name char(30), state_in_hindi text);

postgres=# copy test from 'c:/Pgfile/state_data.txt' with delimiter ',' CSV;
ERROR:  invalid input syntax for integer: "ï»¿9"
CONTEXT:  COPY test, line 1, column state_code: "ï»¿9"

Eg:-
C:\Pgfile>bomremover.exe . *
Added '.\state_data.txt' to processing list.
Press enter to process all files in the list. (1 files in total)

Processing file '.\state_data.txt'...
Finished. Press Enter to Exit

 state_code | state_name | State_name_in_hindi
------------+------------+---------------------
          9 | Karnataka  | αñòαñ░αÑ<8d>αñ¿αñ╛αñƒαñò
         10 | Kerala     | αñòαÑçαñ░αñ│αñ╛
(2 rows)

$ env USE_PGXS=1 pgxnclient install mysql_fdw
INFO: best version: mysql_fdw 1.0.1
INFO: saving /tmp/tmpjrznTj/mysql_fdw-1.0.1.zip
INFO: unpacking: /tmp/tmpjrznTj/mysql_fdw-1.0.1.zip
INFO: building extension
gcc -O2 -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -g -fpic -I/usr/include/mysql -I. -I. -I/home/josh/devel/pg91/include/postgresql/server -I/home/josh/devel/pg91/include/postgresql/internal -D_GNU_SOURCE -I/usr/include/libxml2   -c -o mysql_fdw.o mysql_fdw.c
mysql_fdw.c: In function ‘mysqlPlanForeignScan’:
mysql_fdw.c:466:8: warning: ‘rows’ may be used uninitialized in this function [-Wmaybe-uninitialized]
gcc -O2 -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fexcess-precision=standard -g -fpic -shared -o mysql_fdw.so mysql_fdw.o -L/home/josh/devel/pg91/lib -L/usr/lib  -Wl,--as-needed -Wl,-rpath,'/home/josh/devel/pg91/lib',--enable-new-dtags  -L/usr/lib/x86_64-linux-gnu -lmysqlclient -lpthread -lz -lm -lrt -ldl
INFO: installing extension
< ... snip ... >

for i in `cat mysql_tables`; do
    echo "CREATE FOREIGN TABLE mysql_schema.$i ( ... table definition ...)
        SERVER mysql_server OPTIONS (
            database 'mysqldb',
            query 'SELECT ... some fields ... FROM $i'
        );"
done

mysql> create table bad_dates (mydate date);
Query OK, 0 rows affected (0.07 sec)

mysql> insert into bad_dates values ('2013-02-30'), ('0000-00-00');
Query OK, 2 rows affected (0.02 sec)
Records: 2  Duplicates: 0  Warnings: 0

josh=# create extension mysql_fdw;
CREATE EXTENSION

josh=# create server mysql_svr foreign data wrapper mysql_fdw options (address '127.0.0.1', port '3306');
CREATE SERVER

josh=# create user mapping for public server mysql_svr options (username 'josh', password '');
CREATE USER MAPPING

josh=# create foreign table bad_dates (mydate date) server mysql_svr options (query 'select * from test.bad_dates');
CREATE FOREIGN TABLE

josh=# select * from bad_dates ;
ERROR:  date/time field value out of range: "0000-00-00"

SELECT CASE disabled WHEN '0000-00-00' THEN NULL ELSE disabled END,
    -- various other fields
    FROM some_table

<testxml xmlns:ns1="http://www.example.com" xmlns:ns2="http://www.example2.com">
    <ns1:foo>
        <ns2:bar>baz</ns2:bar>
    </ns1:foo>
</testxml>

SELECT xpath(
    '/testxml/ns1:foo',
    '<testxml xmlns:ns1="http://www.example.com" xmlns:ns2="http://www.example2.com"><ns1:foo><ns2:bar>baz</ns2:bar></ns1:foo></testxml>'::xml,
    ARRAY[
        ['ns1','http://www.example.com'],
        ['ns2','http://www.example2.com']
    ]
);
-- Result: <ns1:foo><ns2:bar>baz</ns2:bar></ns1:foo>

SELECT xpath(
    '/ns1:foo/ns2:bar/text()',
    (xpath(
        '/testxml/ns1:foo',
        '<testxml xmlns:ns1="http://www.example.com" xmlns:ns2="http://www.example2.com"><ns1:foo><ns2:bar>baz</ns2:bar></ns1:foo></testxml>'::xml,
        ARRAY[
            ['ns1','http://www.example.com'],
            ['ns2','http://www.example2.com']
        ]
    ))[1]
);

-- Error:
-- ERROR:  could not create XPath object
-- DETAIL:  namespace error : Namespace prefix ns1 on foo is not defined
-- <ns1:foo>
--         ^
-- namespace error : Namespace prefix ns2 on bar is not defined
--   <ns2:bar>baz</ns2:bar>
--           ^
-- Undefined namespace prefix
-- xmlXPathCompiledEval: evaluation failed

SELECT xpath(
    '/ns1:foo/ns2:bar/text()',
    (xpath(
        '/testxml/ns1:foo',
        '<testxml xmlns:ns1="http://www.example.com" xmlns:ns2="http://www.example2.com"><ns1:foo><ns2:bar>baz</ns2:bar></ns1:foo></testxml>'::xml,
        ARRAY[
            ['ns1','http://www.example.com'],
            ['ns2','http://www.example2.com']
        ]
    ))[1],
    ARRAY[
        ['ns1','http://www.example.com'],
        ['ns2','http://www.example2.com']
    ]
);

-- Returns:
--  xpath 
-- -------
--  {}
-- (1 row)
-- And NOT 'baz' which is what we want.

SELECT xpath_fragment('/ns1:foo/ns2:bar/text()','<ns1:foo><ns2:bar>baz</ns2:bar></ns1:foo>',ARRAY['ns1','ns2']);

 xpath_fragment 
----------------
 {baz}
(1 row)

CREATE OR REPLACE FUNCTION xpath_fragment(_XPath text, _XML xml, _NSNames text[]) RETURNS XML[] AS $BODY$
DECLARE
_ text;
_WrappedXML xml;
_NSArray text[][] := ARRAY[]::text[][];
BEGIN

SELECT ('<xml ' || array_to_string(array_agg('xmlns:' || unnest || '=" "'),' ') || '>' || _XML::text || '</xml>')::text INTO _WrappedXML FROM unnest(_NSNames);

FOR _ IN SELECT unnest(_NSNames)
LOOP
    _NSArray := _NSArray || ARRAY[[_, ' ']];
END LOOP;

RETURN xpath('/xml' || _XPath, _WrappedXML, _NSArray);
END;
$BODY$ LANGUAGE plpgsql IMMUTABLE;

SELECT a.id as a_id, a.properties, sum(b.amount) as expenses
FROM people a
   JOIN expenses b on a.id = b.person_id
GROUP BY a.id, a.properties

WITH exp as 
(
  SELECT a.id as a_id, sum(b.amount) as expenses
  FROM people a
     JOIN expenses b on a.id = b.person_id
  GROUP BY a.id
)
SELECT exp.*, p.properties
FROM exp 
   JOIN people p ON p.id = exp.a_id

Planet PostgreSQL

Bruce Momjian: Video Interview

Josh Berkus: PostgreSQL New Development Priorities 4: Parallel Query

Selena Deckelmann: Distributed databases: a series of posts including 2-phase commit in Postgres

Hubert 'depesz' Lubaczewski: Explaining the unexplainable – part 4

Gabriele Bartolini: Configuring retention policies in Barman

Josh Berkus: PostgreSQL New Development Priorities 3: Pluggable Parser

Selena Deckelmann: Migrations with Alembic: a lightspeed tour

Joe Abbate: Pyrseas contributions solicited

Mason Sharp: Backup and Recovery in Postgres-XC compared to PostgreSQL

Josh Berkus: PostgreSQL New Development Priorities 2: Pluggable Storage

Andrew Dunstan: Our commitment to quality is second to none.

Raghavendra Rao: Disk page checksums to detect filesystem failures in PostgreSQL 9.3Beta 1

Christophe Pettus: DjangoCon US 2013 CFP is open

Hubert 'depesz' Lubaczewski: A tale of automating tests of Pg with Bash

Josh Berkus: Unconference, 9.3 Beta

Pavel Stehule: PostgreSQL history

Andrew Dunstan: PostgreSQL tables for single Redis objects

Michael Paquier: Postgres 9.3 feature highlight: new flavors of IF EXISTS and IF NOT EXISTS

Hans-Juergen Schoenig: Finding similar texts in PostgreSQL

Josh Berkus: PostgreSQL New Development Priorities: Scale It Now

Andrew Dunstan: Redis FDW gets a slightly new home

Raghavendra Rao: What if, import file (txt/csv) having "BOM-ed UTF-8" encoding?

Andrew Dunstan: Sometimes you just get hit by a huge application load

Jignesh Shah: How can PostgreSQL 9.3 beta1 help you?

Bruce Momjian: PgLife Averages Thirty Active Users

Leo Hsu and Regina Obe: PostgreSQL 9.3 extension treats for windows users: plV8

Joshua Tolley: Foreign Data Wrappers

Joel Jacobson: ::xml madness

Leo Hsu and Regina Obe: PostGIS 2.1.0 beta2 is out and windows binaries available

Andrew Dunstan: Only GROUP BY what you really need to